2024-11-13 05:35:18,829 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-13 05:35:18,847 main DEBUG Took 0.014808 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 05:35:18,847 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 05:35:18,848 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 05:35:18,849 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 05:35:18,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,860 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 05:35:18,876 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,878 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,880 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,882 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,882 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,883 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,884 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,885 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,885 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,886 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,887 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,887 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,888 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,889 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,889 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,890 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,890 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,891 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,891 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,892 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:35:18,893 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,894 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 05:35:18,896 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:35:18,898 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 05:35:18,901 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 05:35:18,902 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 05:35:18,904 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 05:35:18,904 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 05:35:18,915 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 05:35:18,919 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 05:35:18,921 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 05:35:18,922 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 05:35:18,922 main DEBUG createAppenders(={Console}) 2024-11-13 05:35:18,923 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-13 05:35:18,924 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-13 05:35:18,924 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-13 05:35:18,925 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 05:35:18,926 main DEBUG OutputStream closed 2024-11-13 05:35:18,926 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 05:35:18,926 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 05:35:18,927 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-13 05:35:19,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 05:35:19,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 05:35:19,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 05:35:19,025 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 05:35:19,026 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 05:35:19,026 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 05:35:19,026 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 05:35:19,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 05:35:19,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 05:35:19,028 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 05:35:19,028 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 05:35:19,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 05:35:19,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 05:35:19,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 05:35:19,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 05:35:19,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 05:35:19,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 05:35:19,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 05:35:19,034 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 05:35:19,035 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-13 05:35:19,035 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 05:35:19,036 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-13T05:35:19,346 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d 2024-11-13 05:35:19,350 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 05:35:19,350 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T05:35:19,362 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-11-13T05:35:19,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=5510 2024-11-13T05:35:19,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:35:19,425 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719, deleteOnExit=true 2024-11-13T05:35:19,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:35:19,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/test.cache.data in system properties and HBase conf 2024-11-13T05:35:19,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:35:19,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:35:19,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:35:19,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:35:19,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:35:19,521 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T05:35:19,621 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:35:19,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:35:19,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:35:19,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:35:19,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:35:19,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:35:19,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:35:19,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:35:19,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:35:19,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:35:19,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:35:19,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:35:19,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:35:19,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:35:19,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:35:20,152 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:35:20,864 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T05:35:20,948 INFO [Time-limited test {}] log.Log(170): Logging initialized @2914ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T05:35:21,037 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:35:21,097 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:35:21,118 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:35:21,118 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:35:21,119 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:35:21,133 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:35:21,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:35:21,137 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:35:21,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6de997b9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/java.io.tmpdir/jetty-localhost-44291-hadoop-hdfs-3_4_1-tests_jar-_-any-17865684862205807807/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:35:21,341 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:44291} 2024-11-13T05:35:21,342 INFO [Time-limited test {}] server.Server(415): Started @3309ms 2024-11-13T05:35:21,376 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:35:21,946 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:35:21,954 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:35:21,955 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:35:21,955 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:35:21,955 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:35:21,956 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3305dd74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:35:21,956 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7893eb07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:35:22,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f93babe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/java.io.tmpdir/jetty-localhost-36249-hadoop-hdfs-3_4_1-tests_jar-_-any-13239861825291564358/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:35:22,064 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@737d6c99{HTTP/1.1, (http/1.1)}{localhost:36249} 2024-11-13T05:35:22,064 INFO [Time-limited test {}] server.Server(415): Started @4032ms 2024-11-13T05:35:22,136 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:35:22,255 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:35:22,260 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:35:22,261 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:35:22,261 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:35:22,261 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:35:22,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ff5148a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:35:22,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25ca9bb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:35:22,390 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c963ecd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/java.io.tmpdir/jetty-localhost-45927-hadoop-hdfs-3_4_1-tests_jar-_-any-12789904751236446006/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:35:22,391 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a10aed{HTTP/1.1, (http/1.1)}{localhost:45927} 2024-11-13T05:35:22,392 INFO [Time-limited test {}] server.Server(415): Started @4359ms 2024-11-13T05:35:22,395 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:35:23,594 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data3/current/BP-1047395441-172.17.0.3-1731476120252/current, will proceed with Du for space computation calculation, 2024-11-13T05:35:23,594 WARN [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data1/current/BP-1047395441-172.17.0.3-1731476120252/current, will proceed with Du for space computation calculation, 2024-11-13T05:35:23,594 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data4/current/BP-1047395441-172.17.0.3-1731476120252/current, will proceed with Du for space computation calculation, 2024-11-13T05:35:23,594 WARN [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data2/current/BP-1047395441-172.17.0.3-1731476120252/current, will proceed with Du for space computation calculation, 2024-11-13T05:35:23,646 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:35:23,647 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:35:23,704 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe58031968344ac98 with lease ID 0x59ac9094b8ce9efd: Processing first storage report for DS-1c6d09a3-6722-4f22-ac6b-42084edaf381 from datanode DatanodeRegistration(127.0.0.1:40959, datanodeUuid=53060c84-a12b-470e-9234-2db78e3bccb3, infoPort=35889, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252) 2024-11-13T05:35:23,705 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe58031968344ac98 with lease ID 0x59ac9094b8ce9efd: from storage DS-1c6d09a3-6722-4f22-ac6b-42084edaf381 node DatanodeRegistration(127.0.0.1:40959, datanodeUuid=53060c84-a12b-470e-9234-2db78e3bccb3, infoPort=35889, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:35:23,706 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd7a969e13eefc866 with lease ID 0x59ac9094b8ce9efc: Processing first storage report for DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d from datanode DatanodeRegistration(127.0.0.1:33703, datanodeUuid=62f16d3c-b4a8-4b41-a1f8-6cd567ed0eb0, infoPort=40267, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252) 2024-11-13T05:35:23,706 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7a969e13eefc866 with lease ID 0x59ac9094b8ce9efc: from storage DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d node DatanodeRegistration(127.0.0.1:33703, datanodeUuid=62f16d3c-b4a8-4b41-a1f8-6cd567ed0eb0, infoPort=40267, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:35:23,706 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe58031968344ac98 with lease ID 0x59ac9094b8ce9efd: Processing first storage report for DS-e7a38456-9420-45c7-affd-399eb215d72e from datanode DatanodeRegistration(127.0.0.1:40959, datanodeUuid=53060c84-a12b-470e-9234-2db78e3bccb3, infoPort=35889, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252) 2024-11-13T05:35:23,706 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe58031968344ac98 with lease ID 0x59ac9094b8ce9efd: from storage DS-e7a38456-9420-45c7-affd-399eb215d72e node DatanodeRegistration(127.0.0.1:40959, datanodeUuid=53060c84-a12b-470e-9234-2db78e3bccb3, infoPort=35889, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:35:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd7a969e13eefc866 with lease ID 0x59ac9094b8ce9efc: Processing first storage report for DS-0b5e972b-ec36-4a00-b3a9-152cc3061fb2 from datanode DatanodeRegistration(127.0.0.1:33703, datanodeUuid=62f16d3c-b4a8-4b41-a1f8-6cd567ed0eb0, infoPort=40267, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252) 2024-11-13T05:35:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7a969e13eefc866 with lease ID 0x59ac9094b8ce9efc: from storage DS-0b5e972b-ec36-4a00-b3a9-152cc3061fb2 node DatanodeRegistration(127.0.0.1:33703, datanodeUuid=62f16d3c-b4a8-4b41-a1f8-6cd567ed0eb0, infoPort=40267, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1922115484;c=1731476120252), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:35:23,744 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d 2024-11-13T05:35:23,817 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/zookeeper_0, clientPort=50305, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:35:23,828 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50305 2024-11-13T05:35:23,841 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:23,844 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:24,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:35:24,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:35:24,457 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c with version=8 2024-11-13T05:35:24,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:35:24,537 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T05:35:24,769 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:35:24,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:35:24,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:35:24,783 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:35:24,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:35:24,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:35:24,919 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:35:24,973 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T05:35:24,982 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T05:35:24,985 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:35:25,009 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 97748 (auto-detected) 2024-11-13T05:35:25,010 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T05:35:25,027 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46731 2024-11-13T05:35:25,051 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46731 connecting to ZooKeeper ensemble=127.0.0.1:50305 2024-11-13T05:35:25,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:467310x0, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:35:25,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46731-0x10132b0ec410000 connected 2024-11-13T05:35:25,458 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:25,460 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:25,470 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:35:25,474 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c, hbase.cluster.distributed=false 2024-11-13T05:35:25,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:35:25,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46731 2024-11-13T05:35:25,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46731 2024-11-13T05:35:25,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46731 2024-11-13T05:35:25,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46731 2024-11-13T05:35:25,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46731 2024-11-13T05:35:25,625 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:35:25,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:35:25,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:35:25,627 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:35:25,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:35:25,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:35:25,630 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:35:25,634 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:35:25,635 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37623 2024-11-13T05:35:25,638 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37623 connecting to ZooKeeper ensemble=127.0.0.1:50305 2024-11-13T05:35:25,640 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:25,647 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:25,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376230x0, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:35:25,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376230x0, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:35:25,677 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37623-0x10132b0ec410001 connected 2024-11-13T05:35:25,682 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:35:25,690 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:35:25,693 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:35:25,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:35:25,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37623 2024-11-13T05:35:25,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37623 2024-11-13T05:35:25,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37623 2024-11-13T05:35:25,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37623 2024-11-13T05:35:25,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37623 2024-11-13T05:35:25,727 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:46731 2024-11-13T05:35:25,728 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,46731,1731476124614 2024-11-13T05:35:25,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:35:25,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:35:25,742 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,46731,1731476124614 2024-11-13T05:35:25,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:35:25,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:25,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:25,773 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:35:25,774 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,46731,1731476124614 from backup master directory 2024-11-13T05:35:25,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:35:25,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,46731,1731476124614 2024-11-13T05:35:25,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:35:25,783 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:35:25,783 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,46731,1731476124614 2024-11-13T05:35:25,785 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T05:35:25,786 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T05:35:25,839 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase.id] with ID: 91127b22-598b-4ea0-a8e7-b40fe9d06d5f 2024-11-13T05:35:25,839 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/.tmp/hbase.id 2024-11-13T05:35:25,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:35:25,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:35:25,854 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/.tmp/hbase.id]:[hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase.id] 2024-11-13T05:35:25,895 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:25,900 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:35:25,918 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T05:35:25,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:25,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:35:26,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:35:26,014 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:35:26,017 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:35:26,024 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:35:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:35:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:35:26,082 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store 2024-11-13T05:35:26,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:35:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:35:26,107 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T05:35:26,111 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:35:26,112 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:35:26,113 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:35:26,113 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:35:26,115 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:35:26,115 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:35:26,115 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:35:26,117 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476126112Disabling compacts and flushes for region at 1731476126112Disabling writes for close at 1731476126115 (+3 ms)Writing region close event to WAL at 1731476126115Closed at 1731476126115 2024-11-13T05:35:26,119 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/.initializing 2024-11-13T05:35:26,119 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/WALs/a61c20bd729c,46731,1731476124614 2024-11-13T05:35:26,140 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C46731%2C1731476124614, suffix=, logDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/WALs/a61c20bd729c,46731,1731476124614, archiveDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/oldWALs, maxLogs=10 2024-11-13T05:35:26,149 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C46731%2C1731476124614.1731476126145 2024-11-13T05:35:26,169 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/WALs/a61c20bd729c,46731,1731476124614/a61c20bd729c%2C46731%2C1731476124614.1731476126145 2024-11-13T05:35:26,183 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:35:26,184 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:35:26,185 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:35:26,189 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,190 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:35:26,260 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:26,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:35:26,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:35:26,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:35:26,271 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:35:26,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:35:26,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:35:26,277 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,281 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,282 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,288 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,288 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,292 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:35:26,296 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:35:26,301 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:35:26,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=784948, jitterRate=-0.0018874257802963257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:35:26,309 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476126203Initializing all the Stores at 1731476126205 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476126206 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476126207 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476126207Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476126207Cleaning up temporary data from old regions at 1731476126289 (+82 ms)Region opened successfully at 1731476126309 (+20 ms) 2024-11-13T05:35:26,311 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:35:26,342 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ac49cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:35:26,371 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:35:26,381 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:35:26,382 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:35:26,384 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:35:26,386 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:35:26,391 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T05:35:26,391 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:35:26,416 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:35:26,424 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:35:26,477 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:35:26,479 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:35:26,481 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:35:26,487 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:35:26,489 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:35:26,493 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:35:26,498 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:35:26,500 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:35:26,508 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:35:26,524 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:35:26,529 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:35:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:35:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:35:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:26,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:26,543 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,46731,1731476124614, sessionid=0x10132b0ec410000, setting cluster-up flag (Was=false) 2024-11-13T05:35:26,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:26,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:26,603 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:35:26,605 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,46731,1731476124614 2024-11-13T05:35:26,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:26,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:26,655 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:35:26,658 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,46731,1731476124614 2024-11-13T05:35:26,665 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:35:26,717 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(746): ClusterId : 91127b22-598b-4ea0-a8e7-b40fe9d06d5f 2024-11-13T05:35:26,719 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:35:26,731 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:35:26,731 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:35:26,741 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:35:26,741 DEBUG [RS:0;a61c20bd729c:37623 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8360ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:35:26,743 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:35:26,752 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:35:26,755 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:37623 2024-11-13T05:35:26,757 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:35:26,759 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:35:26,759 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:35:26,759 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:35:26,762 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,46731,1731476124614 with port=37623, startcode=1731476125586 2024-11-13T05:35:26,763 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,46731,1731476124614 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:35:26,770 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:35:26,770 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:35:26,770 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:35:26,770 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:35:26,771 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:35:26,771 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,771 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:35:26,771 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,777 DEBUG [RS:0;a61c20bd729c:37623 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:35:26,777 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:35:26,778 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:35:26,785 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,786 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:35:26,801 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476156800 2024-11-13T05:35:26,803 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:35:26,804 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:35:26,808 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:35:26,809 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:35:26,809 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:35:26,809 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:35:26,810 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:35:26,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:35:26,816 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:35:26,817 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c 2024-11-13T05:35:26,817 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:35:26,819 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:35:26,819 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:35:26,825 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:35:26,826 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:35:26,828 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476126827,5,FailOnTimeoutGroup] 2024-11-13T05:35:26,834 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476126828,5,FailOnTimeoutGroup] 2024-11-13T05:35:26,834 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,834 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:35:26,837 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,837 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:35:26,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:35:26,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:35:26,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:35:26,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:35:26,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:26,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:35:26,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:35:26,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:26,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:35:26,860 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47847, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:35:26,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:35:26,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:26,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:35:26,867 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46731 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,37623,1731476125586 2024-11-13T05:35:26,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:35:26,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:26,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:26,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:35:26,870 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46731 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,37623,1731476125586 2024-11-13T05:35:26,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740 2024-11-13T05:35:26,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740 2024-11-13T05:35:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:35:26,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:35:26,877 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:35:26,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:35:26,885 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:35:26,887 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=882421, jitterRate=0.12205652892589569}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:35:26,888 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c 2024-11-13T05:35:26,888 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38293 2024-11-13T05:35:26,888 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:35:26,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476126846Initializing all the Stores at 1731476126848 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476126848Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476126848Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476126848Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476126849 (+1 ms)Cleaning up temporary data from old regions at 1731476126876 (+27 ms)Region opened successfully at 1731476126892 (+16 ms) 2024-11-13T05:35:26,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:35:26,893 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:35:26,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:35:26,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:35:26,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:35:26,895 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:35:26,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476126893Disabling compacts and flushes for region at 1731476126893Disabling writes for close at 1731476126894 (+1 ms)Writing region close event to WAL at 1731476126895 (+1 ms)Closed at 1731476126895 2024-11-13T05:35:26,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:35:26,898 DEBUG [RS:0;a61c20bd729c:37623 {}] zookeeper.ZKUtil(111): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,37623,1731476125586 2024-11-13T05:35:26,899 WARN [RS:0;a61c20bd729c:37623 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:35:26,899 INFO [RS:0;a61c20bd729c:37623 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:35:26,899 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:35:26,899 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586 2024-11-13T05:35:26,899 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:35:26,903 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,37623,1731476125586] 2024-11-13T05:35:26,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:35:26,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:35:26,919 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:35:26,926 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:35:26,943 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:35:26,947 INFO [RS:0;a61c20bd729c:37623 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:35:26,947 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,949 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:35:26,954 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:35:26,955 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,956 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,956 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,956 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,956 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,956 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,957 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:35:26,958 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:35:26,958 DEBUG [RS:0;a61c20bd729c:37623 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:35:26,959 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,959 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,959 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,959 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,959 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,959 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,37623,1731476125586-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:35:26,977 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:35:26,979 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,37623,1731476125586-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,979 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,979 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.Replication(171): a61c20bd729c,37623,1731476125586 started 2024-11-13T05:35:26,998 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:26,999 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,37623,1731476125586, RpcServer on a61c20bd729c/172.17.0.3:37623, sessionid=0x10132b0ec410001 2024-11-13T05:35:26,999 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:35:27,000 DEBUG [RS:0;a61c20bd729c:37623 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,37623,1731476125586 2024-11-13T05:35:27,000 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,37623,1731476125586' 2024-11-13T05:35:27,000 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:35:27,001 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:35:27,002 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:35:27,002 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:35:27,003 DEBUG [RS:0;a61c20bd729c:37623 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,37623,1731476125586 2024-11-13T05:35:27,003 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,37623,1731476125586' 2024-11-13T05:35:27,003 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:35:27,004 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:35:27,004 DEBUG [RS:0;a61c20bd729c:37623 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:35:27,004 INFO [RS:0;a61c20bd729c:37623 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:35:27,005 INFO [RS:0;a61c20bd729c:37623 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:35:27,070 WARN [a61c20bd729c:46731 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:35:27,112 INFO [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C37623%2C1731476125586, suffix=, logDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586, archiveDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs, maxLogs=32 2024-11-13T05:35:27,115 INFO [RS:0;a61c20bd729c:37623 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476127114 2024-11-13T05:35:27,125 INFO [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476127114 2024-11-13T05:35:27,126 DEBUG [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:35:27,322 DEBUG [a61c20bd729c:46731 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:35:27,333 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,37623,1731476125586 2024-11-13T05:35:27,338 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,37623,1731476125586, state=OPENING 2024-11-13T05:35:27,392 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:35:27,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:27,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:35:27,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:35:27,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:35:27,405 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:35:27,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,37623,1731476125586}] 2024-11-13T05:35:27,578 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:35:27,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:35:27,594 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:35:27,595 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:35:27,599 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C37623%2C1731476125586.meta, suffix=.meta, logDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586, archiveDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs, maxLogs=32 2024-11-13T05:35:27,602 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.meta.1731476127601.meta 2024-11-13T05:35:27,611 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.meta.1731476127601.meta 2024-11-13T05:35:27,616 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:35:27,620 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:35:27,623 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:35:27,626 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:35:27,632 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:35:27,636 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:35:27,636 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:35:27,637 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:35:27,637 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:35:27,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:35:27,641 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:35:27,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:27,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:27,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:35:27,645 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:35:27,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:27,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:27,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:35:27,648 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:35:27,648 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:27,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:27,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:35:27,651 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:35:27,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:27,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:35:27,652 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:35:27,654 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740 2024-11-13T05:35:27,657 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740 2024-11-13T05:35:27,659 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:35:27,659 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:35:27,660 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:35:27,663 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:35:27,664 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791268, jitterRate=0.0061501264572143555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:35:27,665 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:35:27,666 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476127637Writing region info on filesystem at 1731476127637Initializing all the Stores at 1731476127639 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476127639Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476127639Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476127639Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476127639Cleaning up temporary data from old regions at 1731476127659 (+20 ms)Running coprocessor post-open hooks at 1731476127665 (+6 ms)Region opened successfully at 1731476127666 (+1 ms) 2024-11-13T05:35:27,672 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476127570 2024-11-13T05:35:27,682 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:35:27,683 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:35:27,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,37623,1731476125586 2024-11-13T05:35:27,687 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,37623,1731476125586, state=OPEN 2024-11-13T05:35:27,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:35:27,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:35:27,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:35:27,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:35:27,727 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,37623,1731476125586 2024-11-13T05:35:27,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:35:27,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,37623,1731476125586 in 320 msec 2024-11-13T05:35:27,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:35:27,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 827 msec 2024-11-13T05:35:27,741 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:35:27,741 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:35:27,758 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:35:27,759 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,37623,1731476125586, seqNum=-1] 2024-11-13T05:35:27,782 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:35:27,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38671, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:35:27,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1070 sec 2024-11-13T05:35:27,806 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476127806, completionTime=-1 2024-11-13T05:35:27,808 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:35:27,809 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:35:27,833 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:35:27,833 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476187833 2024-11-13T05:35:27,833 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476247833 2024-11-13T05:35:27,833 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-13T05:35:27,836 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46731,1731476124614-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,837 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46731,1731476124614-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,837 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46731,1731476124614-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,838 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:46731, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,838 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,839 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,846 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:35:27,866 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.082sec 2024-11-13T05:35:27,867 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:35:27,868 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:35:27,869 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:35:27,870 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:35:27,870 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:35:27,871 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46731,1731476124614-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:35:27,872 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46731,1731476124614-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:35:27,880 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:35:27,881 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:35:27,881 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46731,1731476124614-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:35:27,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2961789e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:35:27,930 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T05:35:27,930 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T05:35:27,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,46731,-1 for getting cluster id 2024-11-13T05:35:27,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:35:27,943 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '91127b22-598b-4ea0-a8e7-b40fe9d06d5f' 2024-11-13T05:35:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:35:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "91127b22-598b-4ea0-a8e7-b40fe9d06d5f" 2024-11-13T05:35:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef6e436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:35:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,46731,-1] 2024-11-13T05:35:27,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:35:27,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:35:27,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:35:27,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a039b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:35:27,959 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:35:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,37623,1731476125586, seqNum=-1] 2024-11-13T05:35:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:35:27,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:35:27,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,46731,1731476124614 2024-11-13T05:35:27,997 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:35:28,004 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:35:28,008 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:35:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a61c20bd729c,46731,1731476124614 2024-11-13T05:35:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28c43b06 2024-11-13T05:35:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:35:28,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:35:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-13T05:35:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-13T05:35:28,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:35:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-13T05:35:28,042 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:35:28,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-13T05:35:28,044 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:28,046 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:35:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:35:28,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741835_1011 (size=389) 2024-11-13T05:35:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741835_1011 (size=389) 2024-11-13T05:35:28,088 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6803e9709a5eb9cc40e592a25c8ee113, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c 2024-11-13T05:35:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741836_1012 (size=72) 2024-11-13T05:35:28,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741836_1012 (size=72) 2024-11-13T05:35:28,101 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:35:28,101 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 6803e9709a5eb9cc40e592a25c8ee113, disabling compactions & flushes 2024-11-13T05:35:28,102 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,102 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,102 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. after waiting 0 ms 2024-11-13T05:35:28,102 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,102 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,102 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6803e9709a5eb9cc40e592a25c8ee113: Waiting for close lock at 1731476128101Disabling compacts and flushes for region at 1731476128101Disabling writes for close at 1731476128102 (+1 ms)Writing region close event to WAL at 1731476128102Closed at 1731476128102 2024-11-13T05:35:28,105 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:35:28,111 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731476128105"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476128105"}]},"ts":"1731476128105"} 2024-11-13T05:35:28,117 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:35:28,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:35:28,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476128120"}]},"ts":"1731476128120"} 2024-11-13T05:35:28,129 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-13T05:35:28,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6803e9709a5eb9cc40e592a25c8ee113, ASSIGN}] 2024-11-13T05:35:28,135 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6803e9709a5eb9cc40e592a25c8ee113, ASSIGN 2024-11-13T05:35:28,138 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6803e9709a5eb9cc40e592a25c8ee113, ASSIGN; state=OFFLINE, location=a61c20bd729c,37623,1731476125586; forceNewPlan=false, retain=false 2024-11-13T05:35:28,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6803e9709a5eb9cc40e592a25c8ee113, regionState=OPENING, regionLocation=a61c20bd729c,37623,1731476125586 2024-11-13T05:35:28,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6803e9709a5eb9cc40e592a25c8ee113, ASSIGN because future has completed 2024-11-13T05:35:28,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6803e9709a5eb9cc40e592a25c8ee113, server=a61c20bd729c,37623,1731476125586}] 2024-11-13T05:35:28,456 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,456 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6803e9709a5eb9cc40e592a25c8ee113, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:35:28,457 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,457 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:35:28,457 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,457 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,460 INFO [StoreOpener-6803e9709a5eb9cc40e592a25c8ee113-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,463 INFO [StoreOpener-6803e9709a5eb9cc40e592a25c8ee113-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6803e9709a5eb9cc40e592a25c8ee113 columnFamilyName info 2024-11-13T05:35:28,463 DEBUG [StoreOpener-6803e9709a5eb9cc40e592a25c8ee113-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:35:28,464 INFO [StoreOpener-6803e9709a5eb9cc40e592a25c8ee113-1 {}] regionserver.HStore(327): Store=6803e9709a5eb9cc40e592a25c8ee113/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:35:28,464 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,466 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,467 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,468 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,468 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,471 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,508 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:35:28,509 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6803e9709a5eb9cc40e592a25c8ee113; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789030, jitterRate=0.00330469012260437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:35:28,509 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:28,511 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6803e9709a5eb9cc40e592a25c8ee113: Running coprocessor pre-open hook at 1731476128457Writing region info on filesystem at 1731476128457Initializing all the Stores at 1731476128459 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476128459Cleaning up temporary data from old regions at 1731476128468 (+9 ms)Running coprocessor post-open hooks at 1731476128510 (+42 ms)Region opened successfully at 1731476128511 (+1 ms) 2024-11-13T05:35:28,514 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113., pid=6, masterSystemTime=1731476128449 2024-11-13T05:35:28,523 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,523 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:28,525 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6803e9709a5eb9cc40e592a25c8ee113, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,37623,1731476125586 2024-11-13T05:35:28,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6803e9709a5eb9cc40e592a25c8ee113, server=a61c20bd729c,37623,1731476125586 because future has completed 2024-11-13T05:35:28,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:35:28,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6803e9709a5eb9cc40e592a25c8ee113, server=a61c20bd729c,37623,1731476125586 in 237 msec 2024-11-13T05:35:28,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:35:28,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6803e9709a5eb9cc40e592a25c8ee113, ASSIGN in 406 msec 2024-11-13T05:35:28,542 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:35:28,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476128543"}]},"ts":"1731476128543"} 2024-11-13T05:35:28,547 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-13T05:35:28,548 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:35:28,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 515 msec 2024-11-13T05:35:33,148 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:35:33,197 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:35:33,199 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-13T05:35:34,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:35:34,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:35:34,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-13T05:35:34,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-13T05:35:34,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:35:34,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:35:34,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:35:34,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:35:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:35:38,095 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-13T05:35:38,098 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-13T05:35:38,105 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-13T05:35:38,106 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:35:38,106 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476138106 2024-11-13T05:35:38,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:38,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:38,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:38,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:38,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:38,128 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476127114 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476138106 2024-11-13T05:35:38,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:35:38,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476127114 is not closed yet, will try archiving it next time 2024-11-13T05:35:38,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741833_1009 (size=451) 2024-11-13T05:35:38,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741833_1009 (size=451) 2024-11-13T05:35:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113., hostname=a61c20bd729c,37623,1731476125586, seqNum=2] 2024-11-13T05:35:38,534 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476127114 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476127114 2024-11-13T05:35:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37623 {}] regionserver.HRegion(8855): Flush requested on 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:35:50,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6803e9709a5eb9cc40e592a25c8ee113 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:35:50,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/b483fbc425084ab8a1efe45f1f1e5876 is 1080, key is row0001/info:/1731476138142/Put/seqid=0 2024-11-13T05:35:50,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741838_1014 (size=12509) 2024-11-13T05:35:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741838_1014 (size=12509) 2024-11-13T05:35:50,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/b483fbc425084ab8a1efe45f1f1e5876 2024-11-13T05:35:50,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/b483fbc425084ab8a1efe45f1f1e5876 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876 2024-11-13T05:35:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876, entries=7, sequenceid=11, filesize=12.2 K 2024-11-13T05:35:50,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6803e9709a5eb9cc40e592a25c8ee113 in 152ms, sequenceid=11, compaction requested=false 2024-11-13T05:35:50,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6803e9709a5eb9cc40e592a25c8ee113: 2024-11-13T05:35:53,739 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:35:58,180 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476158180 2024-11-13T05:35:58,390 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:35:58,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:58,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:58,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:58,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:58,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:35:58,391 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476138106 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476158180 2024-11-13T05:35:58,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40267:40267),(127.0.0.1/127.0.0.1:35889:35889)] 2024-11-13T05:35:58,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476138106 is not closed yet, will try archiving it next time 2024-11-13T05:35:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741837_1013 (size=12399) 2024-11-13T05:35:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741837_1013 (size=12399) 2024-11-13T05:35:58,596 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:00,799 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:03,145 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 341 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:05,349 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37623 {}] regionserver.HRegion(8855): Flush requested on 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:36:05,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6803e9709a5eb9cc40e592a25c8ee113 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:36:05,552 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:05,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/93a65d59c93046188f1ecd423c66bbb5 is 1080, key is row0008/info:/1731476152170/Put/seqid=0 2024-11-13T05:36:05,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741840_1016 (size=12509) 2024-11-13T05:36:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741840_1016 (size=12509) 2024-11-13T05:36:05,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/93a65d59c93046188f1ecd423c66bbb5 2024-11-13T05:36:05,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/93a65d59c93046188f1ecd423c66bbb5 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/93a65d59c93046188f1ecd423c66bbb5 2024-11-13T05:36:05,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/93a65d59c93046188f1ecd423c66bbb5, entries=7, sequenceid=21, filesize=12.2 K 2024-11-13T05:36:05,788 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:05,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6803e9709a5eb9cc40e592a25c8ee113 in 438ms, sequenceid=21, compaction requested=false 2024-11-13T05:36:05,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6803e9709a5eb9cc40e592a25c8ee113: 2024-11-13T05:36:05,789 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-11-13T05:36:05,789 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:36:05,789 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876 because midkey is the same as first or last row 2024-11-13T05:36:07,554 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:08,021 INFO [master/a61c20bd729c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T05:36:08,021 INFO [master/a61c20bd729c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T05:36:09,757 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:09,759 WARN [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:09,760 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C37623%2C1731476125586:(num 1731476158180) roll requested 2024-11-13T05:36:09,761 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476169760 2024-11-13T05:36:09,968 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:09,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:09,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:09,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:09,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:09,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:09,969 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476158180 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476169760 2024-11-13T05:36:09,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741839_1015 (size=7739) 2024-11-13T05:36:09,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741839_1015 (size=7739) 2024-11-13T05:36:09,974 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:36:09,974 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476158180 is not closed yet, will try archiving it next time 2024-11-13T05:36:09,974 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476138106 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476138106 2024-11-13T05:36:11,962 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:36:13,457 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6803e9709a5eb9cc40e592a25c8ee113, had cached 0 bytes from a total of 25018 2024-11-13T05:36:14,166 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:36:16,370 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:36:18,629 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 255 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:36:20,631 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:36:20,631 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476180631 2024-11-13T05:36:23,739 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:36:25,640 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:36:25,642 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK], DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK]] 2024-11-13T05:36:25,642 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C37623%2C1731476125586:(num 1731476180631) roll requested 2024-11-13T05:36:25,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:25,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:25,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:25,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:25,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:25,643 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476169760 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476180631 2024-11-13T05:36:25,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741841_1017 (size=4753) 2024-11-13T05:36:25,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741841_1017 (size=4753) 2024-11-13T05:36:25,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40267:40267),(127.0.0.1/127.0.0.1:35889:35889)] 2024-11-13T05:36:25,654 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476185654 2024-11-13T05:36:30,657 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:30,657 WARN [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37623 {}] regionserver.HRegion(8855): Flush requested on 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:36:30,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6803e9709a5eb9cc40e592a25c8ee113 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:36:30,663 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:30,663 WARN [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:32,658 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:36:35,660 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:35,660 WARN [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:35,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:35,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:35,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:35,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:35,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:35,666 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476180631 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476185654 2024-11-13T05:36:35,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/83cebe11f9d84f6d97674587e0adae42 is 1080, key is row0015/info:/1731476167352/Put/seqid=0 2024-11-13T05:36:35,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741842_1018 (size=1569) 2024-11-13T05:36:35,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741842_1018 (size=1569) 2024-11-13T05:36:35,676 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40267:40267),(127.0.0.1/127.0.0.1:35889:35889)] 2024-11-13T05:36:35,676 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476180631 is not closed yet, will try archiving it next time 2024-11-13T05:36:35,677 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C37623%2C1731476125586:(num 1731476185654) roll requested 2024-11-13T05:36:35,677 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476195677 2024-11-13T05:36:35,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741844_1020 (size=12509) 2024-11-13T05:36:35,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741844_1020 (size=12509) 2024-11-13T05:36:35,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/83cebe11f9d84f6d97674587e0adae42 2024-11-13T05:36:35,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/83cebe11f9d84f6d97674587e0adae42 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/83cebe11f9d84f6d97674587e0adae42 2024-11-13T05:36:35,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/83cebe11f9d84f6d97674587e0adae42, entries=7, sequenceid=31, filesize=12.2 K 2024-11-13T05:36:40,688 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:40,688 WARN [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:40,714 INFO [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:40,714 WARN [FSHLog-0-hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c-prefix:a61c20bd729c,37623,1731476125586 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33703,DS-fb1fd903-11b3-49cb-9c04-b28819b5e08d,DISK], DatanodeInfoWithStorage[127.0.0.1:40959,DS-1c6d09a3-6722-4f22-ac6b-42084edaf381,DISK]] 2024-11-13T05:36:40,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6803e9709a5eb9cc40e592a25c8ee113 in 10056ms, sequenceid=31, compaction requested=true 2024-11-13T05:36:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6803e9709a5eb9cc40e592a25c8ee113: 2024-11-13T05:36:40,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-11-13T05:36:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:36:40,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876 because midkey is the same as first or last row 2024-11-13T05:36:40,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,715 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476185654 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476195677 2024-11-13T05:36:40,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6803e9709a5eb9cc40e592a25c8ee113:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:36:40,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741843_1019 (size=438) 2024-11-13T05:36:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:36:40,723 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476158180 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476158180 2024-11-13T05:36:40,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741843_1019 (size=438) 2024-11-13T05:36:40,724 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:36:40,724 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C37623%2C1731476125586:(num 1731476195677) roll requested 2024-11-13T05:36:40,726 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476169760 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476169760 2024-11-13T05:36:40,726 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:36:40,729 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476200729 2024-11-13T05:36:40,729 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476180631 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476180631 2024-11-13T05:36:40,730 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:36:40,733 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.HStore(1541): 6803e9709a5eb9cc40e592a25c8ee113/info is initiating minor compaction (all files) 2024-11-13T05:36:40,733 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476185654 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476185654 2024-11-13T05:36:40,733 INFO [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6803e9709a5eb9cc40e592a25c8ee113/info in TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:36:40,734 INFO [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/93a65d59c93046188f1ecd423c66bbb5, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/83cebe11f9d84f6d97674587e0adae42] into tmpdir=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp, totalSize=36.6 K 2024-11-13T05:36:40,736 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] compactions.Compactor(225): Compacting b483fbc425084ab8a1efe45f1f1e5876, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731476138142 2024-11-13T05:36:40,737 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a65d59c93046188f1ecd423c66bbb5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731476152170 2024-11-13T05:36:40,739 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83cebe11f9d84f6d97674587e0adae42, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731476167352 2024-11-13T05:36:40,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,752 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476195677 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476200729 2024-11-13T05:36:40,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741845_1021 (size=93) 2024-11-13T05:36:40,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741845_1021 (size=93) 2024-11-13T05:36:40,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:36:40,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476195677 is not closed yet, will try archiving it next time 2024-11-13T05:36:40,756 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37623%2C1731476125586.1731476200756 2024-11-13T05:36:40,756 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476195677 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs/a61c20bd729c%2C37623%2C1731476125586.1731476195677 2024-11-13T05:36:40,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:36:40,784 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476200729 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476200756 2024-11-13T05:36:40,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741846_1022 (size=1258) 2024-11-13T05:36:40,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741846_1022 (size=1258) 2024-11-13T05:36:40,791 INFO [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6803e9709a5eb9cc40e592a25c8ee113#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:36:40,793 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/58eaeee99b1e4aa282bcb939d8be06c4 is 1080, key is row0001/info:/1731476138142/Put/seqid=0 2024-11-13T05:36:40,797 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35889:35889),(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-13T05:36:40,797 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/WALs/a61c20bd729c,37623,1731476125586/a61c20bd729c%2C37623%2C1731476125586.1731476200729 is not closed yet, will try archiving it next time 2024-11-13T05:36:40,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741848_1024 (size=27710) 2024-11-13T05:36:40,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741848_1024 (size=27710) 2024-11-13T05:36:40,819 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/58eaeee99b1e4aa282bcb939d8be06c4 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/58eaeee99b1e4aa282bcb939d8be06c4 2024-11-13T05:36:40,849 INFO [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6803e9709a5eb9cc40e592a25c8ee113/info of 6803e9709a5eb9cc40e592a25c8ee113 into 58eaeee99b1e4aa282bcb939d8be06c4(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:36:40,849 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6803e9709a5eb9cc40e592a25c8ee113: 2024-11-13T05:36:40,852 INFO [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113., storeName=6803e9709a5eb9cc40e592a25c8ee113/info, priority=13, startTime=1731476200716; duration=0sec 2024-11-13T05:36:40,853 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-13T05:36:40,853 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:36:40,853 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/58eaeee99b1e4aa282bcb939d8be06c4 because midkey is the same as first or last row 2024-11-13T05:36:40,854 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-13T05:36:40,854 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:36:40,854 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/58eaeee99b1e4aa282bcb939d8be06c4 because midkey is the same as first or last row 2024-11-13T05:36:40,855 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-13T05:36:40,855 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:36:40,855 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/58eaeee99b1e4aa282bcb939d8be06c4 because midkey is the same as first or last row 2024-11-13T05:36:40,856 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:36:40,856 DEBUG [RS:0;a61c20bd729c:37623-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6803e9709a5eb9cc40e592a25c8ee113:info 2024-11-13T05:36:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37623 {}] regionserver.HRegion(8855): Flush requested on 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:36:52,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6803e9709a5eb9cc40e592a25c8ee113 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:36:52,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/4a5913854cbf49958c74353f5098d979 is 1080, key is row0022/info:/1731476200758/Put/seqid=0 2024-11-13T05:36:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741849_1025 (size=12509) 2024-11-13T05:36:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741849_1025 (size=12509) 2024-11-13T05:36:52,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/4a5913854cbf49958c74353f5098d979 2024-11-13T05:36:52,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/4a5913854cbf49958c74353f5098d979 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/4a5913854cbf49958c74353f5098d979 2024-11-13T05:36:52,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/4a5913854cbf49958c74353f5098d979, entries=7, sequenceid=42, filesize=12.2 K 2024-11-13T05:36:52,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6803e9709a5eb9cc40e592a25c8ee113 in 43ms, sequenceid=42, compaction requested=false 2024-11-13T05:36:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6803e9709a5eb9cc40e592a25c8ee113: 2024-11-13T05:36:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-11-13T05:36:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:36:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/58eaeee99b1e4aa282bcb939d8be06c4 because midkey is the same as first or last row 2024-11-13T05:36:53,740 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:36:58,458 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6803e9709a5eb9cc40e592a25c8ee113, had cached 0 bytes from a total of 40219 2024-11-13T05:37:00,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:37:00,805 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:37:00,806 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:00,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:00,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:00,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:37:00,813 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:37:00,813 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=245640440, stopped=false 2024-11-13T05:37:00,814 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,46731,1731476124614 2024-11-13T05:37:00,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:00,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:00,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:00,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:00,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:37:00,862 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:00,862 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:37:00,862 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:00,862 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:00,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:00,863 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,37623,1731476125586' ***** 2024-11-13T05:37:00,863 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:37:00,863 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:37:00,863 INFO [RS:0;a61c20bd729c:37623 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:37:00,863 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:37:00,863 INFO [RS:0;a61c20bd729c:37623 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:37:00,864 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(3091): Received CLOSE for 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:37:00,864 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,37623,1731476125586 2024-11-13T05:37:00,864 INFO [RS:0;a61c20bd729c:37623 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:00,864 INFO [RS:0;a61c20bd729c:37623 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:37623. 2024-11-13T05:37:00,864 DEBUG [RS:0;a61c20bd729c:37623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:00,864 DEBUG [RS:0;a61c20bd729c:37623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:00,864 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6803e9709a5eb9cc40e592a25c8ee113, disabling compactions & flushes 2024-11-13T05:37:00,865 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:37:00,865 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:37:00,865 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. after waiting 0 ms 2024-11-13T05:37:00,865 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:37:00,865 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:37:00,865 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:37:00,865 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:37:00,865 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:37:00,865 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6803e9709a5eb9cc40e592a25c8ee113 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-13T05:37:00,865 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:37:00,865 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:37:00,866 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6803e9709a5eb9cc40e592a25c8ee113=TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.} 2024-11-13T05:37:00,866 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:37:00,866 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:37:00,866 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:37:00,866 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:37:00,866 DEBUG [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6803e9709a5eb9cc40e592a25c8ee113 2024-11-13T05:37:00,866 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-13T05:37:00,872 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/ab949ac21f5c49c38348ee54c6b68ecd is 1080, key is row0029/info:/1731476214791/Put/seqid=0 2024-11-13T05:37:00,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741850_1026 (size=8193) 2024-11-13T05:37:00,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741850_1026 (size=8193) 2024-11-13T05:37:00,887 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/ab949ac21f5c49c38348ee54c6b68ecd 2024-11-13T05:37:00,892 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/info/813f0f945ad944f2a6a65581ae3d3f24 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113./info:regioninfo/1731476128525/Put/seqid=0 2024-11-13T05:37:00,897 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/.tmp/info/ab949ac21f5c49c38348ee54c6b68ecd as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/ab949ac21f5c49c38348ee54c6b68ecd 2024-11-13T05:37:00,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741851_1027 (size=7016) 2024-11-13T05:37:00,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741851_1027 (size=7016) 2024-11-13T05:37:00,900 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/info/813f0f945ad944f2a6a65581ae3d3f24 2024-11-13T05:37:00,906 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/ab949ac21f5c49c38348ee54c6b68ecd, entries=3, sequenceid=48, filesize=8.0 K 2024-11-13T05:37:00,908 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 6803e9709a5eb9cc40e592a25c8ee113 in 43ms, sequenceid=48, compaction requested=true 2024-11-13T05:37:00,908 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/93a65d59c93046188f1ecd423c66bbb5, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/83cebe11f9d84f6d97674587e0adae42] to archive 2024-11-13T05:37:00,911 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:37:00,915 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/b483fbc425084ab8a1efe45f1f1e5876 2024-11-13T05:37:00,917 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/93a65d59c93046188f1ecd423c66bbb5 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/93a65d59c93046188f1ecd423c66bbb5 2024-11-13T05:37:00,919 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/83cebe11f9d84f6d97674587e0adae42 to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/info/83cebe11f9d84f6d97674587e0adae42 2024-11-13T05:37:00,930 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/ns/eaf48ad3bffe4d809edac8858803e662 is 43, key is default/ns:d/1731476127789/Put/seqid=0 2024-11-13T05:37:00,932 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a61c20bd729c:46731 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-13T05:37:00,935 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [b483fbc425084ab8a1efe45f1f1e5876=12509, 93a65d59c93046188f1ecd423c66bbb5=12509, 83cebe11f9d84f6d97674587e0adae42=12509] 2024-11-13T05:37:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741852_1028 (size=5153) 2024-11-13T05:37:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741852_1028 (size=5153) 2024-11-13T05:37:00,941 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/ns/eaf48ad3bffe4d809edac8858803e662 2024-11-13T05:37:00,942 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/default/TestLogRolling-testSlowSyncLogRolling/6803e9709a5eb9cc40e592a25c8ee113/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-11-13T05:37:00,944 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:37:00,945 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6803e9709a5eb9cc40e592a25c8ee113: Waiting for close lock at 1731476220864Running coprocessor pre-close hooks at 1731476220864Disabling compacts and flushes for region at 1731476220864Disabling writes for close at 1731476220865 (+1 ms)Obtaining lock to block concurrent updates at 1731476220865Preparing flush snapshotting stores in 6803e9709a5eb9cc40e592a25c8ee113 at 1731476220865Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731476220866 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. at 1731476220867 (+1 ms)Flushing 6803e9709a5eb9cc40e592a25c8ee113/info: creating writer at 1731476220867Flushing 6803e9709a5eb9cc40e592a25c8ee113/info: appending metadata at 1731476220871 (+4 ms)Flushing 6803e9709a5eb9cc40e592a25c8ee113/info: closing flushed file at 1731476220872 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a5d6054: reopening flushed file at 1731476220895 (+23 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 6803e9709a5eb9cc40e592a25c8ee113 in 43ms, sequenceid=48, compaction requested=true at 1731476220908 (+13 ms)Writing region close event to WAL at 1731476220936 (+28 ms)Running coprocessor post-close hooks at 1731476220943 (+7 ms)Closed at 1731476220944 (+1 ms) 2024-11-13T05:37:00,945 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731476128024.6803e9709a5eb9cc40e592a25c8ee113. 2024-11-13T05:37:00,959 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:37:00,959 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:37:00,964 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:00,967 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/table/1d58c04055f64f32b3cb11e10b3884f0 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731476128543/Put/seqid=0 2024-11-13T05:37:00,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741853_1029 (size=5396) 2024-11-13T05:37:00,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741853_1029 (size=5396) 2024-11-13T05:37:00,974 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/table/1d58c04055f64f32b3cb11e10b3884f0 2024-11-13T05:37:00,983 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/info/813f0f945ad944f2a6a65581ae3d3f24 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/info/813f0f945ad944f2a6a65581ae3d3f24 2024-11-13T05:37:00,993 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/info/813f0f945ad944f2a6a65581ae3d3f24, entries=10, sequenceid=11, filesize=6.9 K 2024-11-13T05:37:00,994 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/ns/eaf48ad3bffe4d809edac8858803e662 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/ns/eaf48ad3bffe4d809edac8858803e662 2024-11-13T05:37:01,003 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/ns/eaf48ad3bffe4d809edac8858803e662, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T05:37:01,005 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/.tmp/table/1d58c04055f64f32b3cb11e10b3884f0 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/table/1d58c04055f64f32b3cb11e10b3884f0 2024-11-13T05:37:01,014 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/table/1d58c04055f64f32b3cb11e10b3884f0, entries=2, sequenceid=11, filesize=5.3 K 2024-11-13T05:37:01,016 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 149ms, sequenceid=11, compaction requested=false 2024-11-13T05:37:01,023 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T05:37:01,024 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:37:01,024 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:01,024 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476220865Running coprocessor pre-close hooks at 1731476220865Disabling compacts and flushes for region at 1731476220865Disabling writes for close at 1731476220866 (+1 ms)Obtaining lock to block concurrent updates at 1731476220866Preparing flush snapshotting stores in 1588230740 at 1731476220866Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731476220866Flushing stores of hbase:meta,,1.1588230740 at 1731476220869 (+3 ms)Flushing 1588230740/info: creating writer at 1731476220869Flushing 1588230740/info: appending metadata at 1731476220891 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731476220891Flushing 1588230740/ns: creating writer at 1731476220908 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731476220929 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731476220929Flushing 1588230740/table: creating writer at 1731476220951 (+22 ms)Flushing 1588230740/table: appending metadata at 1731476220967 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731476220967Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d650c57: reopening flushed file at 1731476220982 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78b8311a: reopening flushed file at 1731476220993 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4637b99a: reopening flushed file at 1731476221004 (+11 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 149ms, sequenceid=11, compaction requested=false at 1731476221016 (+12 ms)Writing region close event to WAL at 1731476221018 (+2 ms)Running coprocessor post-close hooks at 1731476221023 (+5 ms)Closed at 1731476221024 (+1 ms) 2024-11-13T05:37:01,024 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:01,066 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,37623,1731476125586; all regions closed. 2024-11-13T05:37:01,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741834_1010 (size=3066) 2024-11-13T05:37:01,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741834_1010 (size=3066) 2024-11-13T05:37:01,081 DEBUG [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs 2024-11-13T05:37:01,081 INFO [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C37623%2C1731476125586.meta:.meta(num 1731476127601) 2024-11-13T05:37:01,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741847_1023 (size=12695) 2024-11-13T05:37:01,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741847_1023 (size=12695) 2024-11-13T05:37:01,088 DEBUG [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/oldWALs 2024-11-13T05:37:01,088 INFO [RS:0;a61c20bd729c:37623 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C37623%2C1731476125586:(num 1731476200756) 2024-11-13T05:37:01,088 DEBUG [RS:0;a61c20bd729c:37623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:01,089 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:01,089 INFO [RS:0;a61c20bd729c:37623 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:01,089 INFO [RS:0;a61c20bd729c:37623 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:01,089 INFO [RS:0;a61c20bd729c:37623 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:01,089 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:01,089 INFO [RS:0;a61c20bd729c:37623 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37623 2024-11-13T05:37:01,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,37623,1731476125586 2024-11-13T05:37:01,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:01,103 INFO [RS:0;a61c20bd729c:37623 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:01,104 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,37623,1731476125586] 2024-11-13T05:37:01,124 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,37623,1731476125586 already deleted, retry=false 2024-11-13T05:37:01,124 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,37623,1731476125586 expired; onlineServers=0 2024-11-13T05:37:01,125 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,46731,1731476124614' ***** 2024-11-13T05:37:01,125 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:37:01,125 INFO [M:0;a61c20bd729c:46731 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:01,125 INFO [M:0;a61c20bd729c:46731 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:01,125 DEBUG [M:0;a61c20bd729c:46731 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:37:01,126 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:37:01,126 DEBUG [M:0;a61c20bd729c:46731 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:37:01,126 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476126828 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476126828,5,FailOnTimeoutGroup] 2024-11-13T05:37:01,126 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476126827 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476126827,5,FailOnTimeoutGroup] 2024-11-13T05:37:01,126 INFO [M:0;a61c20bd729c:46731 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:01,126 INFO [M:0;a61c20bd729c:46731 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:01,127 DEBUG [M:0;a61c20bd729c:46731 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:37:01,127 INFO [M:0;a61c20bd729c:46731 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:37:01,127 INFO [M:0;a61c20bd729c:46731 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:37:01,128 INFO [M:0;a61c20bd729c:46731 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:37:01,128 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:37:01,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:37:01,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:01,135 DEBUG [M:0;a61c20bd729c:46731 {}] zookeeper.ZKUtil(347): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:37:01,135 WARN [M:0;a61c20bd729c:46731 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:37:01,136 INFO [M:0;a61c20bd729c:46731 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/.lastflushedseqids 2024-11-13T05:37:01,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741854_1030 (size=130) 2024-11-13T05:37:01,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741854_1030 (size=130) 2024-11-13T05:37:01,149 INFO [M:0;a61c20bd729c:46731 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:37:01,149 INFO [M:0;a61c20bd729c:46731 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:37:01,150 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:37:01,150 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:01,150 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:01,150 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:37:01,150 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:01,150 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.01 KB heapSize=29.18 KB 2024-11-13T05:37:01,169 DEBUG [M:0;a61c20bd729c:46731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0880b9ca0df34ac199d4eb37557ed6cf is 82, key is hbase:meta,,1/info:regioninfo/1731476127684/Put/seqid=0 2024-11-13T05:37:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741855_1031 (size=5672) 2024-11-13T05:37:01,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741855_1031 (size=5672) 2024-11-13T05:37:01,175 INFO [M:0;a61c20bd729c:46731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0880b9ca0df34ac199d4eb37557ed6cf 2024-11-13T05:37:01,200 DEBUG [M:0;a61c20bd729c:46731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cbf421f011324bcba6bff690bc8ec4d7 is 765, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731476128550/Put/seqid=0 2024-11-13T05:37:01,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741856_1032 (size=6246) 2024-11-13T05:37:01,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741856_1032 (size=6246) 2024-11-13T05:37:01,206 INFO [M:0;a61c20bd729c:46731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.41 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cbf421f011324bcba6bff690bc8ec4d7 2024-11-13T05:37:01,213 INFO [M:0;a61c20bd729c:46731 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cbf421f011324bcba6bff690bc8ec4d7 2024-11-13T05:37:01,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:01,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37623-0x10132b0ec410001, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:01,215 INFO [RS:0;a61c20bd729c:37623 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:01,215 INFO [RS:0;a61c20bd729c:37623 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,37623,1731476125586; zookeeper connection closed. 2024-11-13T05:37:01,215 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@484bae9b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@484bae9b 2024-11-13T05:37:01,216 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:37:01,229 DEBUG [M:0;a61c20bd729c:46731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec2f40e9e5244fb0810dc29aaa1d074c is 69, key is a61c20bd729c,37623,1731476125586/rs:state/1731476126873/Put/seqid=0 2024-11-13T05:37:01,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741857_1033 (size=5156) 2024-11-13T05:37:01,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741857_1033 (size=5156) 2024-11-13T05:37:01,236 INFO [M:0;a61c20bd729c:46731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec2f40e9e5244fb0810dc29aaa1d074c 2024-11-13T05:37:01,257 DEBUG [M:0;a61c20bd729c:46731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d38aa4e26e2d4ebfa3e80bd9abb4c53c is 52, key is load_balancer_on/state:d/1731476128001/Put/seqid=0 2024-11-13T05:37:01,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741858_1034 (size=5056) 2024-11-13T05:37:01,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741858_1034 (size=5056) 2024-11-13T05:37:01,264 INFO [M:0;a61c20bd729c:46731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d38aa4e26e2d4ebfa3e80bd9abb4c53c 2024-11-13T05:37:01,274 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0880b9ca0df34ac199d4eb37557ed6cf as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0880b9ca0df34ac199d4eb37557ed6cf 2024-11-13T05:37:01,281 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0880b9ca0df34ac199d4eb37557ed6cf, entries=8, sequenceid=59, filesize=5.5 K 2024-11-13T05:37:01,284 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cbf421f011324bcba6bff690bc8ec4d7 as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cbf421f011324bcba6bff690bc8ec4d7 2024-11-13T05:37:01,296 INFO [M:0;a61c20bd729c:46731 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cbf421f011324bcba6bff690bc8ec4d7 2024-11-13T05:37:01,296 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cbf421f011324bcba6bff690bc8ec4d7, entries=6, sequenceid=59, filesize=6.1 K 2024-11-13T05:37:01,298 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec2f40e9e5244fb0810dc29aaa1d074c as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec2f40e9e5244fb0810dc29aaa1d074c 2024-11-13T05:37:01,305 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec2f40e9e5244fb0810dc29aaa1d074c, entries=1, sequenceid=59, filesize=5.0 K 2024-11-13T05:37:01,307 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d38aa4e26e2d4ebfa3e80bd9abb4c53c as hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d38aa4e26e2d4ebfa3e80bd9abb4c53c 2024-11-13T05:37:01,315 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d38aa4e26e2d4ebfa3e80bd9abb4c53c, entries=1, sequenceid=59, filesize=4.9 K 2024-11-13T05:37:01,317 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false 2024-11-13T05:37:01,319 INFO [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:01,319 DEBUG [M:0;a61c20bd729c:46731 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476221150Disabling compacts and flushes for region at 1731476221150Disabling writes for close at 1731476221150Obtaining lock to block concurrent updates at 1731476221150Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476221150Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23564, getHeapSize=29816, getOffHeapSize=0, getCellsCount=70 at 1731476221151 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476221152 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476221152Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476221168 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476221168Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476221182 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476221199 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476221199Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476221213 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476221229 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476221229Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476221242 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476221257 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476221257Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c2f1e2: reopening flushed file at 1731476221272 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bda813: reopening flushed file at 1731476221282 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a40d967: reopening flushed file at 1731476221297 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c39d4b0: reopening flushed file at 1731476221305 (+8 ms)Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false at 1731476221317 (+12 ms)Writing region close event to WAL at 1731476221318 (+1 ms)Closed at 1731476221318 2024-11-13T05:37:01,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33703 is added to blk_1073741830_1006 (size=27961) 2024-11-13T05:37:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741830_1006 (size=27961) 2024-11-13T05:37:01,324 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:01,324 INFO [M:0;a61c20bd729c:46731 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:37:01,324 INFO [M:0;a61c20bd729c:46731 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46731 2024-11-13T05:37:01,324 INFO [M:0;a61c20bd729c:46731 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:01,435 INFO [M:0;a61c20bd729c:46731 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:01,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:01,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46731-0x10132b0ec410000, quorum=127.0.0.1:50305, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:01,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c963ecd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:01,443 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a10aed{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:01,443 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:01,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25ca9bb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:01,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ff5148a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:01,450 WARN [BP-1047395441-172.17.0.3-1731476120252 heartbeating to localhost/127.0.0.1:38293 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:01,450 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:01,450 WARN [BP-1047395441-172.17.0.3-1731476120252 heartbeating to localhost/127.0.0.1:38293 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1047395441-172.17.0.3-1731476120252 (Datanode Uuid 53060c84-a12b-470e-9234-2db78e3bccb3) service to localhost/127.0.0.1:38293 2024-11-13T05:37:01,450 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:01,451 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data3/current/BP-1047395441-172.17.0.3-1731476120252 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:01,451 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data4/current/BP-1047395441-172.17.0.3-1731476120252 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:01,452 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:01,457 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f93babe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:01,457 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@737d6c99{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:01,458 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:01,458 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7893eb07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:01,458 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3305dd74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:01,460 WARN [BP-1047395441-172.17.0.3-1731476120252 heartbeating to localhost/127.0.0.1:38293 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:01,460 WARN [BP-1047395441-172.17.0.3-1731476120252 heartbeating to localhost/127.0.0.1:38293 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1047395441-172.17.0.3-1731476120252 (Datanode Uuid 62f16d3c-b4a8-4b41-a1f8-6cd567ed0eb0) service to localhost/127.0.0.1:38293 2024-11-13T05:37:01,460 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:01,460 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:01,461 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data1/current/BP-1047395441-172.17.0.3-1731476120252 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:01,461 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/cluster_1db04f9f-4f0d-2a6c-c4d6-7d131d7f8719/data/data2/current/BP-1047395441-172.17.0.3-1731476120252 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:01,462 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:01,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6de997b9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:37:01,470 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:01,470 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:01,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:01,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:01,504 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:37:01,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:37:01,564 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38293 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:38293 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38293 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@1ab9ef81 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a61c20bd729c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:38293 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:38293 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38293 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: regionserver/a61c20bd729c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a61c20bd729c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38293 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:38293 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=188 (was 262), ProcessCount=11 (was 11), AvailableMemoryMB=2817 (was 5510) 2024-11-13T05:37:01,577 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=188, ProcessCount=11, AvailableMemoryMB=2814 2024-11-13T05:37:01,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:37:01,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.log.dir so I do NOT create it in target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601 2024-11-13T05:37:01,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61026b3c-8968-4e95-20b4-cc3fb868116d/hadoop.tmp.dir so I do NOT create it in target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601 2024-11-13T05:37:01,579 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2, deleteOnExit=true 2024-11-13T05:37:01,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:37:01,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/test.cache.data in system properties and HBase conf 2024-11-13T05:37:01,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:37:01,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:37:01,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:37:01,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:37:01,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:37:01,581 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:37:01,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:37:01,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:37:01,597 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:37:01,966 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:01,974 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:01,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:01,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:01,975 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:01,977 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:01,978 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c8f0dfe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:01,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@495543c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:02,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2787d4b6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/java.io.tmpdir/jetty-localhost-36783-hadoop-hdfs-3_4_1-tests_jar-_-any-3752924666005433689/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:37:02,076 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6f450072{HTTP/1.1, (http/1.1)}{localhost:36783} 2024-11-13T05:37:02,076 INFO [Time-limited test {}] server.Server(415): Started @104044ms 2024-11-13T05:37:02,088 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:37:02,347 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:02,351 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:02,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:02,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:02,352 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:37:02,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a3a779{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:02,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4bedb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:02,451 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a843c6e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/java.io.tmpdir/jetty-localhost-46545-hadoop-hdfs-3_4_1-tests_jar-_-any-16179183720298922807/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:02,452 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17783943{HTTP/1.1, (http/1.1)}{localhost:46545} 2024-11-13T05:37:02,452 INFO [Time-limited test {}] server.Server(415): Started @104420ms 2024-11-13T05:37:02,454 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:02,492 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:02,495 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:02,496 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:02,496 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:02,496 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:37:02,497 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c5ed954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:02,497 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1db3e806{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:02,606 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@513d1177{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/java.io.tmpdir/jetty-localhost-37665-hadoop-hdfs-3_4_1-tests_jar-_-any-10843551040575523329/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:02,606 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c0a2992{HTTP/1.1, (http/1.1)}{localhost:37665} 2024-11-13T05:37:02,606 INFO [Time-limited test {}] server.Server(415): Started @104574ms 2024-11-13T05:37:02,608 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:03,638 WARN [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data1/current/BP-1380223524-172.17.0.3-1731476221608/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:03,638 WARN [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data2/current/BP-1380223524-172.17.0.3-1731476221608/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:03,660 WARN [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:03,662 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c0676d633c62714 with lease ID 0x757d4e0c1e5e5888: Processing first storage report for DS-526aec01-8a0d-44e3-bb36-fb72da9406bd from datanode DatanodeRegistration(127.0.0.1:46287, datanodeUuid=a0a79936-f61e-4251-8831-40a5fea5bf46, infoPort=33653, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608) 2024-11-13T05:37:03,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c0676d633c62714 with lease ID 0x757d4e0c1e5e5888: from storage DS-526aec01-8a0d-44e3-bb36-fb72da9406bd node DatanodeRegistration(127.0.0.1:46287, datanodeUuid=a0a79936-f61e-4251-8831-40a5fea5bf46, infoPort=33653, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:03,663 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c0676d633c62714 with lease ID 0x757d4e0c1e5e5888: Processing first storage report for DS-4ee5fb86-5129-424c-9904-717a1b687a48 from datanode DatanodeRegistration(127.0.0.1:46287, datanodeUuid=a0a79936-f61e-4251-8831-40a5fea5bf46, infoPort=33653, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608) 2024-11-13T05:37:03,663 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c0676d633c62714 with lease ID 0x757d4e0c1e5e5888: from storage DS-4ee5fb86-5129-424c-9904-717a1b687a48 node DatanodeRegistration(127.0.0.1:46287, datanodeUuid=a0a79936-f61e-4251-8831-40a5fea5bf46, infoPort=33653, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:03,762 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data3/current/BP-1380223524-172.17.0.3-1731476221608/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:03,762 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data4/current/BP-1380223524-172.17.0.3-1731476221608/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:03,780 WARN [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c039194d8b08458 with lease ID 0x757d4e0c1e5e5889: Processing first storage report for DS-9ddb4e85-79f1-45af-a891-51b1c8ff80bd from datanode DatanodeRegistration(127.0.0.1:37731, datanodeUuid=b2665710-fc93-4dfa-b61c-ba1c5b08d4eb, infoPort=39129, infoSecurePort=0, ipcPort=46833, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608) 2024-11-13T05:37:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c039194d8b08458 with lease ID 0x757d4e0c1e5e5889: from storage DS-9ddb4e85-79f1-45af-a891-51b1c8ff80bd node DatanodeRegistration(127.0.0.1:37731, datanodeUuid=b2665710-fc93-4dfa-b61c-ba1c5b08d4eb, infoPort=39129, infoSecurePort=0, ipcPort=46833, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c039194d8b08458 with lease ID 0x757d4e0c1e5e5889: Processing first storage report for DS-190cc6d5-e171-4ce3-9bf7-adefcedad548 from datanode DatanodeRegistration(127.0.0.1:37731, datanodeUuid=b2665710-fc93-4dfa-b61c-ba1c5b08d4eb, infoPort=39129, infoSecurePort=0, ipcPort=46833, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608) 2024-11-13T05:37:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c039194d8b08458 with lease ID 0x757d4e0c1e5e5889: from storage DS-190cc6d5-e171-4ce3-9bf7-adefcedad548 node DatanodeRegistration(127.0.0.1:37731, datanodeUuid=b2665710-fc93-4dfa-b61c-ba1c5b08d4eb, infoPort=39129, infoSecurePort=0, ipcPort=46833, storageInfo=lv=-57;cid=testClusterID;nsid=1310157148;c=1731476221608), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:03,846 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601 2024-11-13T05:37:03,849 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/zookeeper_0, clientPort=54179, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:37:03,850 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54179 2024-11-13T05:37:03,850 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:03,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:03,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:03,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:03,865 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5 with version=8 2024-11-13T05:37:03,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:37:03,867 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:37:03,867 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:03,868 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41261 2024-11-13T05:37:03,870 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41261 connecting to ZooKeeper ensemble=127.0.0.1:54179 2024-11-13T05:37:03,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412610x0, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:03,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41261-0x10132b273130000 connected 2024-11-13T05:37:04,045 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:04,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:04,050 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:04,051 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5, hbase.cluster.distributed=false 2024-11-13T05:37:04,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:04,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41261 2024-11-13T05:37:04,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41261 2024-11-13T05:37:04,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41261 2024-11-13T05:37:04,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41261 2024-11-13T05:37:04,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41261 2024-11-13T05:37:04,077 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:37:04,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:04,079 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41617 2024-11-13T05:37:04,082 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41617 connecting to ZooKeeper ensemble=127.0.0.1:54179 2024-11-13T05:37:04,083 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:04,086 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:04,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416170x0, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:04,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:416170x0, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:04,104 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:37:04,108 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:37:04,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:416170x0, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:37:04,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:416170x0, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:04,111 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41617-0x10132b273130001 connected 2024-11-13T05:37:04,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41617 2024-11-13T05:37:04,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41617 2024-11-13T05:37:04,117 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41617 2024-11-13T05:37:04,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41617 2024-11-13T05:37:04,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41617 2024-11-13T05:37:04,145 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:41261 2024-11-13T05:37:04,145 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:04,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:04,156 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:37:04,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,167 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:37:04,167 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,41261,1731476223866 from backup master directory 2024-11-13T05:37:04,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:04,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:04,176 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:37:04,177 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,182 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/hbase.id] with ID: 62b54842-bceb-4d88-84fc-f3a4596eb111 2024-11-13T05:37:04,182 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/.tmp/hbase.id 2024-11-13T05:37:04,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:37:04,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:37:04,189 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/.tmp/hbase.id]:[hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/hbase.id] 2024-11-13T05:37:04,203 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:04,203 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:37:04,205 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-13T05:37:04,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:04,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:04,220 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:37:04,221 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:37:04,222 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:37:04,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:37:04,231 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store 2024-11-13T05:37:04,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:04,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:04,241 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:04,241 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:37:04,241 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:04,241 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:04,241 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:37:04,241 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:04,241 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:04,241 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476224241Disabling compacts and flushes for region at 1731476224241Disabling writes for close at 1731476224241Writing region close event to WAL at 1731476224241Closed at 1731476224241 2024-11-13T05:37:04,242 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/.initializing 2024-11-13T05:37:04,243 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/WALs/a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,246 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C41261%2C1731476223866, suffix=, logDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/WALs/a61c20bd729c,41261,1731476223866, archiveDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/oldWALs, maxLogs=10 2024-11-13T05:37:04,247 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C41261%2C1731476223866.1731476224247 2024-11-13T05:37:04,254 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/WALs/a61c20bd729c,41261,1731476223866/a61c20bd729c%2C41261%2C1731476223866.1731476224247 2024-11-13T05:37:04,255 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33653:33653),(127.0.0.1/127.0.0.1:39129:39129)] 2024-11-13T05:37:04,256 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:37:04,256 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:04,256 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,256 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:37:04,263 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,264 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:04,264 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:37:04,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:04,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:37:04,269 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:04,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:37:04,272 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:04,273 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,274 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,275 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,277 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,277 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,278 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:37:04,280 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:04,283 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:04,283 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=777513, jitterRate=-0.011342152953147888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:37:04,285 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476224257Initializing all the Stores at 1731476224258 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476224258Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476224260 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476224260Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476224260Cleaning up temporary data from old regions at 1731476224277 (+17 ms)Region opened successfully at 1731476224285 (+8 ms) 2024-11-13T05:37:04,288 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:37:04,295 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49140b54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:04,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:37:04,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:37:04,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:37:04,297 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:37:04,298 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:37:04,298 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:37:04,298 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:37:04,303 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:37:04,304 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:37:04,313 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:37:04,314 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:37:04,315 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:37:04,324 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:37:04,324 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:37:04,326 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:37:04,334 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:37:04,335 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:37:04,345 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:37:04,347 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:37:04,360 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:37:04,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:04,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:04,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,371 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,41261,1731476223866, sessionid=0x10132b273130000, setting cluster-up flag (Was=false) 2024-11-13T05:37:04,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,423 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:37:04,425 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:04,476 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:37:04,480 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,41261,1731476223866 2024-11-13T05:37:04,484 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:37:04,487 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:04,487 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:37:04,488 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:37:04,488 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,41261,1731476223866 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:37:04,490 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,491 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476254491 2024-11-13T05:37:04,491 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,492 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:37:04,492 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:04,493 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:37:04,493 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:37:04,493 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:37:04,493 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:37:04,493 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:37:04,493 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476224493,5,FailOnTimeoutGroup] 2024-11-13T05:37:04,494 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476224493,5,FailOnTimeoutGroup] 2024-11-13T05:37:04,494 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,494 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:37:04,494 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,494 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,494 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,494 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:37:04,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:37:04,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:37:04,505 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:37:04,505 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5 2024-11-13T05:37:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:04,517 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:04,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:37:04,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:37:04,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:04,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:37:04,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:37:04,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:04,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:37:04,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:37:04,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:04,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:37:04,529 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(746): ClusterId : 62b54842-bceb-4d88-84fc-f3a4596eb111 2024-11-13T05:37:04,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:37:04,529 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:37:04,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:04,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:04,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:37:04,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740 2024-11-13T05:37:04,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740 2024-11-13T05:37:04,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:37:04,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:37:04,534 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:37:04,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:37:04,537 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:04,538 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=758872, jitterRate=-0.0350455641746521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:37:04,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476224517Initializing all the Stores at 1731476224518 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476224518Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476224519 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476224519Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476224519Cleaning up temporary data from old regions at 1731476224533 (+14 ms)Region opened successfully at 1731476224539 (+6 ms) 2024-11-13T05:37:04,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:37:04,539 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:37:04,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:37:04,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:37:04,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:37:04,540 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:37:04,540 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:37:04,540 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:04,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476224539Disabling compacts and flushes for region at 1731476224539Disabling writes for close at 1731476224539Writing region close event to WAL at 1731476224540 (+1 ms)Closed at 1731476224540 2024-11-13T05:37:04,541 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:04,541 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:37:04,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:37:04,543 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:37:04,545 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:37:04,551 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:37:04,551 DEBUG [RS:0;a61c20bd729c:41617 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2501f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:04,563 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:41617 2024-11-13T05:37:04,563 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:37:04,563 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:37:04,563 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:37:04,564 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,41261,1731476223866 with port=41617, startcode=1731476224077 2024-11-13T05:37:04,565 DEBUG [RS:0;a61c20bd729c:41617 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:37:04,568 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38155, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:37:04,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41261 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,569 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41261 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,571 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5 2024-11-13T05:37:04,571 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36205 2024-11-13T05:37:04,571 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:37:04,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:04,582 DEBUG [RS:0;a61c20bd729c:41617 {}] zookeeper.ZKUtil(111): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,582 WARN [RS:0;a61c20bd729c:41617 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:37:04,582 INFO [RS:0;a61c20bd729c:41617 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:04,582 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/WALs/a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,582 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,41617,1731476224077] 2024-11-13T05:37:04,586 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:37:04,589 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:37:04,589 INFO [RS:0;a61c20bd729c:41617 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:37:04,589 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,589 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:37:04,590 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:37:04,591 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,591 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,592 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,592 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:04,592 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:37:04,592 DEBUG [RS:0;a61c20bd729c:41617 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:37:04,594 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,594 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,594 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,595 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,595 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,595 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41617,1731476224077-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:37:04,610 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:37:04,610 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41617,1731476224077-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,610 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,610 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.Replication(171): a61c20bd729c,41617,1731476224077 started 2024-11-13T05:37:04,626 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:04,626 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,41617,1731476224077, RpcServer on a61c20bd729c/172.17.0.3:41617, sessionid=0x10132b273130001 2024-11-13T05:37:04,626 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:37:04,627 DEBUG [RS:0;a61c20bd729c:41617 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,627 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,41617,1731476224077' 2024-11-13T05:37:04,627 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:37:04,627 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:37:04,628 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:37:04,628 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:37:04,628 DEBUG [RS:0;a61c20bd729c:41617 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,628 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,41617,1731476224077' 2024-11-13T05:37:04,628 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:37:04,628 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:37:04,629 DEBUG [RS:0;a61c20bd729c:41617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:37:04,629 INFO [RS:0;a61c20bd729c:41617 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:37:04,629 INFO [RS:0;a61c20bd729c:41617 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:37:04,695 WARN [a61c20bd729c:41261 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:37:04,732 INFO [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C41617%2C1731476224077, suffix=, logDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/WALs/a61c20bd729c,41617,1731476224077, archiveDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/oldWALs, maxLogs=32 2024-11-13T05:37:04,734 INFO [RS:0;a61c20bd729c:41617 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C41617%2C1731476224077.1731476224734 2024-11-13T05:37:04,741 INFO [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/WALs/a61c20bd729c,41617,1731476224077/a61c20bd729c%2C41617%2C1731476224077.1731476224734 2024-11-13T05:37:04,746 DEBUG [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39129:39129),(127.0.0.1/127.0.0.1:33653:33653)] 2024-11-13T05:37:04,945 DEBUG [a61c20bd729c:41261 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:37:04,947 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,41617,1731476224077 2024-11-13T05:37:04,952 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,41617,1731476224077, state=OPENING 2024-11-13T05:37:04,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:37:04,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:37:04,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-13T05:37:05,012 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:37:05,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:05,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:05,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:05,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:05,024 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:37:05,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,41617,1731476224077}] 2024-11-13T05:37:05,178 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:37:05,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46211, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:37:05,191 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:37:05,191 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:05,195 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C41617%2C1731476224077.meta, suffix=.meta, logDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/WALs/a61c20bd729c,41617,1731476224077, archiveDir=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/oldWALs, maxLogs=32 2024-11-13T05:37:05,198 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C41617%2C1731476224077.meta.1731476225197.meta 2024-11-13T05:37:05,205 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/WALs/a61c20bd729c,41617,1731476224077/a61c20bd729c%2C41617%2C1731476224077.meta.1731476225197.meta 2024-11-13T05:37:05,205 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33653:33653),(127.0.0.1/127.0.0.1:39129:39129)] 2024-11-13T05:37:05,206 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:37:05,206 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:37:05,206 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:37:05,207 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:37:05,207 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:37:05,207 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:05,207 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:37:05,207 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:37:05,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:37:05,209 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:37:05,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:05,210 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:05,210 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:37:05,211 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:37:05,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:05,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:05,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:37:05,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:37:05,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:05,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:05,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:37:05,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:37:05,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:05,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:05,215 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:37:05,216 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740 2024-11-13T05:37:05,217 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740 2024-11-13T05:37:05,218 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:37:05,219 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:37:05,219 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:37:05,221 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:37:05,221 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=810054, jitterRate=0.030037641525268555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:37:05,222 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:37:05,223 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476225207Writing region info on filesystem at 1731476225207Initializing all the Stores at 1731476225208 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476225208Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476225208Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476225208Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476225208Cleaning up temporary data from old regions at 1731476225219 (+11 ms)Running coprocessor post-open hooks at 1731476225222 (+3 ms)Region opened successfully at 1731476225223 (+1 ms) 2024-11-13T05:37:05,224 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476225178 2024-11-13T05:37:05,227 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:37:05,227 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:37:05,228 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,41617,1731476224077 2024-11-13T05:37:05,229 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,41617,1731476224077, state=OPEN 2024-11-13T05:37:05,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:37:05,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:37:05,267 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,41617,1731476224077 2024-11-13T05:37:05,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:05,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:05,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:37:05,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,41617,1731476224077 in 243 msec 2024-11-13T05:37:05,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:37:05,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 730 msec 2024-11-13T05:37:05,276 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:05,276 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:37:05,278 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:37:05,278 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,41617,1731476224077, seqNum=-1] 2024-11-13T05:37:05,278 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:37:05,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57489, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:37:05,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 801 msec 2024-11-13T05:37:05,292 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476225292, completionTime=-1 2024-11-13T05:37:05,292 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:37:05,292 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:37:05,295 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:37:05,295 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476285295 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476345296 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41261,1731476223866-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41261,1731476223866-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41261,1731476223866-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:41261, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,296 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,297 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,299 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.125sec 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41261,1731476223866-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:37:05,302 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41261,1731476223866-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:37:05,305 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:37:05,305 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:37:05,306 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41261,1731476223866-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:05,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d21920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:37:05,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,41261,-1 for getting cluster id 2024-11-13T05:37:05,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:37:05,332 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '62b54842-bceb-4d88-84fc-f3a4596eb111' 2024-11-13T05:37:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:37:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "62b54842-bceb-4d88-84fc-f3a4596eb111" 2024-11-13T05:37:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408353f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:37:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,41261,-1] 2024-11-13T05:37:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:37:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:05,337 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:37:05,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b23b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:37:05,339 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:37:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,41617,1731476224077, seqNum=-1] 2024-11-13T05:37:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:37:05,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:37:05,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,41261,1731476223866 2024-11-13T05:37:05,346 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:05,349 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:37:05,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:37:05,349 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:37:05,350 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:05,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:05,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:05,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:37:05,350 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:37:05,350 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1434811321, stopped=false 2024-11-13T05:37:05,350 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,41261,1731476223866 2024-11-13T05:37:05,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:05,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:05,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:37:05,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:05,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:05,371 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:37:05,371 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:05,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:05,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:05,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:05,371 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,41617,1731476224077' ***** 2024-11-13T05:37:05,371 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:37:05,372 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,41617,1731476224077 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:41617. 2024-11-13T05:37:05,372 DEBUG [RS:0;a61c20bd729c:41617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:05,372 DEBUG [RS:0;a61c20bd729c:41617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:37:05,372 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:37:05,373 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:37:05,373 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:37:05,373 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T05:37:05,373 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:37:05,373 DEBUG [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:37:05,373 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:37:05,373 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:37:05,373 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:37:05,373 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:37:05,374 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:37:05,374 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-13T05:37:05,398 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/.tmp/ns/ce4f601663e14f439cd7bdf890a63655 is 43, key is default/ns:d/1731476225281/Put/seqid=0 2024-11-13T05:37:05,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741835_1011 (size=5153) 2024-11-13T05:37:05,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741835_1011 (size=5153) 2024-11-13T05:37:05,405 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/.tmp/ns/ce4f601663e14f439cd7bdf890a63655 2024-11-13T05:37:05,414 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/.tmp/ns/ce4f601663e14f439cd7bdf890a63655 as hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/ns/ce4f601663e14f439cd7bdf890a63655 2024-11-13T05:37:05,423 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/ns/ce4f601663e14f439cd7bdf890a63655, entries=2, sequenceid=6, filesize=5.0 K 2024-11-13T05:37:05,424 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false 2024-11-13T05:37:05,424 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:37:05,433 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T05:37:05,434 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:37:05,434 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:05,434 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476225373Running coprocessor pre-close hooks at 1731476225373Disabling compacts and flushes for region at 1731476225373Disabling writes for close at 1731476225373Obtaining lock to block concurrent updates at 1731476225374 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731476225374Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731476225374Flushing stores of hbase:meta,,1.1588230740 at 1731476225375 (+1 ms)Flushing 1588230740/ns: creating writer at 1731476225375Flushing 1588230740/ns: appending metadata at 1731476225397 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731476225397Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39be6a9: reopening flushed file at 1731476225413 (+16 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false at 1731476225424 (+11 ms)Writing region close event to WAL at 1731476225426 (+2 ms)Running coprocessor post-close hooks at 1731476225434 (+8 ms)Closed at 1731476225434 2024-11-13T05:37:05,435 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:05,573 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,41617,1731476224077; all regions closed. 2024-11-13T05:37:05,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741834_1010 (size=1152) 2024-11-13T05:37:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741834_1010 (size=1152) 2024-11-13T05:37:05,583 DEBUG [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/oldWALs 2024-11-13T05:37:05,583 INFO [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C41617%2C1731476224077.meta:.meta(num 1731476225197) 2024-11-13T05:37:05,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741833_1009 (size=93) 2024-11-13T05:37:05,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741833_1009 (size=93) 2024-11-13T05:37:05,590 DEBUG [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/oldWALs 2024-11-13T05:37:05,590 INFO [RS:0;a61c20bd729c:41617 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C41617%2C1731476224077:(num 1731476224734) 2024-11-13T05:37:05,590 DEBUG [RS:0;a61c20bd729c:41617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:05,590 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:05,591 INFO [RS:0;a61c20bd729c:41617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:05,591 INFO [RS:0;a61c20bd729c:41617 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:05,591 INFO [RS:0;a61c20bd729c:41617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:05,591 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:05,591 INFO [RS:0;a61c20bd729c:41617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41617 2024-11-13T05:37:05,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:05,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,41617,1731476224077 2024-11-13T05:37:05,615 INFO [RS:0;a61c20bd729c:41617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:05,615 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,41617,1731476224077] 2024-11-13T05:37:05,639 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,41617,1731476224077 already deleted, retry=false 2024-11-13T05:37:05,639 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,41617,1731476224077 expired; onlineServers=0 2024-11-13T05:37:05,639 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,41261,1731476223866' ***** 2024-11-13T05:37:05,640 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:37:05,640 INFO [M:0;a61c20bd729c:41261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:05,640 INFO [M:0;a61c20bd729c:41261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:05,640 DEBUG [M:0;a61c20bd729c:41261 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:37:05,640 DEBUG [M:0;a61c20bd729c:41261 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:37:05,640 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:37:05,640 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476224493 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476224493,5,FailOnTimeoutGroup] 2024-11-13T05:37:05,640 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476224493 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476224493,5,FailOnTimeoutGroup] 2024-11-13T05:37:05,640 INFO [M:0;a61c20bd729c:41261 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:05,640 INFO [M:0;a61c20bd729c:41261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:05,641 DEBUG [M:0;a61c20bd729c:41261 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:37:05,641 INFO [M:0;a61c20bd729c:41261 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:37:05,641 INFO [M:0;a61c20bd729c:41261 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:37:05,641 INFO [M:0;a61c20bd729c:41261 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:37:05,641 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:37:05,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:37:05,650 DEBUG [M:0;a61c20bd729c:41261 {}] zookeeper.ZKUtil(347): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:37:05,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:05,650 WARN [M:0;a61c20bd729c:41261 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:37:05,651 INFO [M:0;a61c20bd729c:41261 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/.lastflushedseqids 2024-11-13T05:37:05,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741836_1012 (size=99) 2024-11-13T05:37:05,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741836_1012 (size=99) 2024-11-13T05:37:05,659 INFO [M:0;a61c20bd729c:41261 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:37:05,659 INFO [M:0;a61c20bd729c:41261 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:37:05,659 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:37:05,659 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:05,659 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:05,659 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:37:05,659 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:05,659 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-13T05:37:05,682 DEBUG [M:0;a61c20bd729c:41261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/390615a84bbd4328972ac347092ade8e is 82, key is hbase:meta,,1/info:regioninfo/1731476225228/Put/seqid=0 2024-11-13T05:37:05,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741837_1013 (size=5672) 2024-11-13T05:37:05,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741837_1013 (size=5672) 2024-11-13T05:37:05,689 INFO [M:0;a61c20bd729c:41261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/390615a84bbd4328972ac347092ade8e 2024-11-13T05:37:05,713 DEBUG [M:0;a61c20bd729c:41261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a237295757464d66b72ae98c5de82308 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731476225288/Put/seqid=0 2024-11-13T05:37:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741838_1014 (size=5275) 2024-11-13T05:37:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741838_1014 (size=5275) 2024-11-13T05:37:05,720 INFO [M:0;a61c20bd729c:41261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a237295757464d66b72ae98c5de82308 2024-11-13T05:37:05,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:05,729 INFO [RS:0;a61c20bd729c:41617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:05,729 INFO [RS:0;a61c20bd729c:41617 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,41617,1731476224077; zookeeper connection closed. 2024-11-13T05:37:05,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41617-0x10132b273130001, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:05,730 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d5512f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d5512f2 2024-11-13T05:37:05,730 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:37:05,744 DEBUG [M:0;a61c20bd729c:41261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7366638f05314cdd855c6192ef9f0b97 is 69, key is a61c20bd729c,41617,1731476224077/rs:state/1731476224569/Put/seqid=0 2024-11-13T05:37:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741839_1015 (size=5156) 2024-11-13T05:37:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741839_1015 (size=5156) 2024-11-13T05:37:05,751 INFO [M:0;a61c20bd729c:41261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7366638f05314cdd855c6192ef9f0b97 2024-11-13T05:37:05,776 DEBUG [M:0;a61c20bd729c:41261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/642d37b29128467196e296e3961e0f4a is 52, key is load_balancer_on/state:d/1731476225348/Put/seqid=0 2024-11-13T05:37:05,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741840_1016 (size=5056) 2024-11-13T05:37:05,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741840_1016 (size=5056) 2024-11-13T05:37:05,782 INFO [M:0;a61c20bd729c:41261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/642d37b29128467196e296e3961e0f4a 2024-11-13T05:37:05,791 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/390615a84bbd4328972ac347092ade8e as hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/390615a84bbd4328972ac347092ade8e 2024-11-13T05:37:05,798 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/390615a84bbd4328972ac347092ade8e, entries=8, sequenceid=29, filesize=5.5 K 2024-11-13T05:37:05,799 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a237295757464d66b72ae98c5de82308 as hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a237295757464d66b72ae98c5de82308 2024-11-13T05:37:05,809 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a237295757464d66b72ae98c5de82308, entries=3, sequenceid=29, filesize=5.2 K 2024-11-13T05:37:05,811 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7366638f05314cdd855c6192ef9f0b97 as hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7366638f05314cdd855c6192ef9f0b97 2024-11-13T05:37:05,818 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7366638f05314cdd855c6192ef9f0b97, entries=1, sequenceid=29, filesize=5.0 K 2024-11-13T05:37:05,819 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/642d37b29128467196e296e3961e0f4a as hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/642d37b29128467196e296e3961e0f4a 2024-11-13T05:37:05,827 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36205/user/jenkins/test-data/19d6920e-3b6b-dfb7-68b2-8b295311d7f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/642d37b29128467196e296e3961e0f4a, entries=1, sequenceid=29, filesize=4.9 K 2024-11-13T05:37:05,828 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=29, compaction requested=false 2024-11-13T05:37:05,830 INFO [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:05,830 DEBUG [M:0;a61c20bd729c:41261 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476225659Disabling compacts and flushes for region at 1731476225659Disabling writes for close at 1731476225659Obtaining lock to block concurrent updates at 1731476225659Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476225659Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731476225660 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476225661 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476225661Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476225681 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476225681Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476225696 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476225712 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476225712Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476225727 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476225744 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476225744Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476225759 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476225775 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476225775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bed28cb: reopening flushed file at 1731476225790 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fc8d31a: reopening flushed file at 1731476225798 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d166664: reopening flushed file at 1731476225810 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@de4692a: reopening flushed file at 1731476225818 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 169ms, sequenceid=29, compaction requested=false at 1731476225828 (+10 ms)Writing region close event to WAL at 1731476225830 (+2 ms)Closed at 1731476225830 2024-11-13T05:37:05,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:05,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37731 is added to blk_1073741830_1006 (size=10311) 2024-11-13T05:37:05,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741830_1006 (size=10311) 2024-11-13T05:37:05,835 INFO [M:0;a61c20bd729c:41261 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:37:05,835 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:05,835 INFO [M:0;a61c20bd729c:41261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41261 2024-11-13T05:37:05,835 INFO [M:0;a61c20bd729c:41261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:05,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:05,944 INFO [M:0;a61c20bd729c:41261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:05,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41261-0x10132b273130000, quorum=127.0.0.1:54179, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:05,947 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:05,955 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:05,976 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@513d1177{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:05,976 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c0a2992{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:05,976 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:05,977 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1db3e806{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:05,977 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c5ed954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:05,978 WARN [BP-1380223524-172.17.0.3-1731476221608 heartbeating to localhost/127.0.0.1:36205 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:05,978 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:05,978 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:05,978 WARN [BP-1380223524-172.17.0.3-1731476221608 heartbeating to localhost/127.0.0.1:36205 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1380223524-172.17.0.3-1731476221608 (Datanode Uuid b2665710-fc93-4dfa-b61c-ba1c5b08d4eb) service to localhost/127.0.0.1:36205 2024-11-13T05:37:05,979 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data3/current/BP-1380223524-172.17.0.3-1731476221608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:05,979 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data4/current/BP-1380223524-172.17.0.3-1731476221608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:05,979 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:05,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a843c6e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:05,982 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17783943{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:05,982 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:05,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4bedb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:05,983 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a3a779{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:05,984 WARN [BP-1380223524-172.17.0.3-1731476221608 heartbeating to localhost/127.0.0.1:36205 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:05,984 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:05,984 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:05,984 WARN [BP-1380223524-172.17.0.3-1731476221608 heartbeating to localhost/127.0.0.1:36205 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1380223524-172.17.0.3-1731476221608 (Datanode Uuid a0a79936-f61e-4251-8831-40a5fea5bf46) service to localhost/127.0.0.1:36205 2024-11-13T05:37:05,985 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data1/current/BP-1380223524-172.17.0.3-1731476221608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:05,985 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/cluster_a6cb3c7b-ecf4-1da0-3666-ea936a8ab1b2/data/data2/current/BP-1380223524-172.17.0.3-1731476221608 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:05,985 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:05,997 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2787d4b6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:37:05,998 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6f450072{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:05,998 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:05,998 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@495543c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:05,998 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c8f0dfe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:06,006 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.log.dir so I do NOT create it in target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41c17d24-0422-4523-fcbb-6f1a17fab601/hadoop.tmp.dir so I do NOT create it in target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604, deleteOnExit=true 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:37:06,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/test.cache.data in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:37:06,024 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:37:06,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:37:06,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:37:06,038 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:37:06,320 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T05:37:06,322 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:06,336 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:06,338 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:06,339 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:06,361 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:06,366 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:06,368 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:06,368 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:06,368 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:37:06,369 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:06,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@100caf4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:06,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c3e366f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:06,469 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@acc902f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-40209-hadoop-hdfs-3_4_1-tests_jar-_-any-8948058202558191480/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:37:06,472 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71ccb409{HTTP/1.1, (http/1.1)}{localhost:40209} 2024-11-13T05:37:06,473 INFO [Time-limited test {}] server.Server(415): Started @108440ms 2024-11-13T05:37:06,485 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:37:06,595 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:06,739 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:06,743 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:06,743 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:06,743 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:06,743 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:06,744 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ad2b72e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:06,744 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ed9c958{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:06,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@527a79d3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-37167-hadoop-hdfs-3_4_1-tests_jar-_-any-10290491075151920041/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:06,840 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78ac40e9{HTTP/1.1, (http/1.1)}{localhost:37167} 2024-11-13T05:37:06,840 INFO [Time-limited test {}] server.Server(415): Started @108808ms 2024-11-13T05:37:06,842 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:06,871 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:06,874 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:06,875 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:06,875 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:06,876 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:06,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f186434{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:06,877 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d227195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:06,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6cc9fc90{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-44753-hadoop-hdfs-3_4_1-tests_jar-_-any-625872341995936650/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:06,976 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7909691f{HTTP/1.1, (http/1.1)}{localhost:44753} 2024-11-13T05:37:06,976 INFO [Time-limited test {}] server.Server(415): Started @108944ms 2024-11-13T05:37:06,978 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:08,247 WARN [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data1/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:08,247 WARN [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data2/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:08,270 WARN [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:08,275 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc2f2d3ee702b7350 with lease ID 0xbb59ec497afcae0c: Processing first storage report for DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58 from datanode DatanodeRegistration(127.0.0.1:36475, datanodeUuid=ae4362a4-1e1a-4b20-aee5-e9c92751a9a0, infoPort=40671, infoSecurePort=0, ipcPort=45147, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:08,275 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2f2d3ee702b7350 with lease ID 0xbb59ec497afcae0c: from storage DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58 node DatanodeRegistration(127.0.0.1:36475, datanodeUuid=ae4362a4-1e1a-4b20-aee5-e9c92751a9a0, infoPort=40671, infoSecurePort=0, ipcPort=45147, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:08,275 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc2f2d3ee702b7350 with lease ID 0xbb59ec497afcae0c: Processing first storage report for DS-d0715957-7cde-41f5-a3f0-c2b330024d13 from datanode DatanodeRegistration(127.0.0.1:36475, datanodeUuid=ae4362a4-1e1a-4b20-aee5-e9c92751a9a0, infoPort=40671, infoSecurePort=0, ipcPort=45147, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:08,275 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2f2d3ee702b7350 with lease ID 0xbb59ec497afcae0c: from storage DS-d0715957-7cde-41f5-a3f0-c2b330024d13 node DatanodeRegistration(127.0.0.1:36475, datanodeUuid=ae4362a4-1e1a-4b20-aee5-e9c92751a9a0, infoPort=40671, infoSecurePort=0, ipcPort=45147, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:37:08,384 WARN [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data3/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:08,384 WARN [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data4/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:08,407 WARN [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9024f08c6fe74247 with lease ID 0xbb59ec497afcae0d: Processing first storage report for DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f from datanode DatanodeRegistration(127.0.0.1:33291, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45421, infoSecurePort=0, ipcPort=42837, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9024f08c6fe74247 with lease ID 0xbb59ec497afcae0d: from storage DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f node DatanodeRegistration(127.0.0.1:33291, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45421, infoSecurePort=0, ipcPort=42837, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9024f08c6fe74247 with lease ID 0xbb59ec497afcae0d: Processing first storage report for DS-700af77e-f81e-4f89-90c1-372cdd8cffd5 from datanode DatanodeRegistration(127.0.0.1:33291, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45421, infoSecurePort=0, ipcPort=42837, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:08,410 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9024f08c6fe74247 with lease ID 0xbb59ec497afcae0d: from storage DS-700af77e-f81e-4f89-90c1-372cdd8cffd5 node DatanodeRegistration(127.0.0.1:33291, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45421, infoSecurePort=0, ipcPort=42837, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:08,430 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0 2024-11-13T05:37:08,433 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/zookeeper_0, clientPort=51491, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:37:08,434 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51491 2024-11-13T05:37:08,435 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,437 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:08,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:08,449 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817 with version=8 2024-11-13T05:37:08,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:37:08,451 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:08,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:08,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:08,451 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:08,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:08,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:08,451 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:37:08,452 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:08,452 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45157 2024-11-13T05:37:08,454 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45157 connecting to ZooKeeper ensemble=127.0.0.1:51491 2024-11-13T05:37:08,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:451570x0, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:08,528 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45157-0x10132b284fc0000 connected 2024-11-13T05:37:08,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,616 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:08,621 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817, hbase.cluster.distributed=false 2024-11-13T05:37:08,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:08,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45157 2024-11-13T05:37:08,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45157 2024-11-13T05:37:08,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45157 2024-11-13T05:37:08,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45157 2024-11-13T05:37:08,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45157 2024-11-13T05:37:08,646 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:08,646 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:08,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:08,647 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:08,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:08,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:08,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:37:08,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:08,648 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44475 2024-11-13T05:37:08,650 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44475 connecting to ZooKeeper ensemble=127.0.0.1:51491 2024-11-13T05:37:08,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,653 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,665 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444750x0, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:08,665 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444750x0, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:08,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44475-0x10132b284fc0001 connected 2024-11-13T05:37:08,666 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:37:08,667 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:37:08,668 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:37:08,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:08,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44475 2024-11-13T05:37:08,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44475 2024-11-13T05:37:08,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44475 2024-11-13T05:37:08,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44475 2024-11-13T05:37:08,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44475 2024-11-13T05:37:08,691 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:45157 2024-11-13T05:37:08,691 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,45157,1731476228451 2024-11-13T05:37:08,702 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:08,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:08,703 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,45157,1731476228451 2024-11-13T05:37:08,713 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:37:08,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:08,713 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:08,714 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:37:08,714 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,45157,1731476228451 from backup master directory 2024-11-13T05:37:08,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,45157,1731476228451 2024-11-13T05:37:08,723 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:08,723 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:37:08,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:08,724 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,45157,1731476228451 2024-11-13T05:37:08,729 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/hbase.id] with ID: 107b19c6-4e72-447a-a76d-be515e236e6a 2024-11-13T05:37:08,729 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/.tmp/hbase.id 2024-11-13T05:37:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:37:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:37:08,737 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/.tmp/hbase.id]:[hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/hbase.id] 2024-11-13T05:37:08,751 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:08,751 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:37:08,753 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-13T05:37:08,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:08,766 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:08,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:08,774 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:37:08,775 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:37:08,775 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:37:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:37:08,785 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store 2024-11-13T05:37:08,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:08,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:08,794 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:08,794 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:37:08,794 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:08,794 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:08,794 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:37:08,794 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:08,794 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:08,794 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476228794Disabling compacts and flushes for region at 1731476228794Disabling writes for close at 1731476228794Writing region close event to WAL at 1731476228794Closed at 1731476228794 2024-11-13T05:37:08,795 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/.initializing 2024-11-13T05:37:08,796 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451 2024-11-13T05:37:08,799 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C45157%2C1731476228451, suffix=, logDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451, archiveDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/oldWALs, maxLogs=10 2024-11-13T05:37:08,799 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C45157%2C1731476228451.1731476228799 2024-11-13T05:37:08,805 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 2024-11-13T05:37:08,807 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40671:40671),(127.0.0.1/127.0.0.1:45421:45421)] 2024-11-13T05:37:08,808 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:37:08,808 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:08,808 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,808 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:37:08,812 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:08,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:08,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:37:08,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:08,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:08,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:37:08,818 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:08,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:08,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:37:08,820 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:08,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:08,821 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,822 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,822 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,823 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,823 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,824 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:37:08,825 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:08,827 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:08,828 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754043, jitterRate=-0.041185975074768066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:37:08,829 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476228808Initializing all the Stores at 1731476228810 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476228810Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476228810Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476228810Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476228810Cleaning up temporary data from old regions at 1731476228823 (+13 ms)Region opened successfully at 1731476228829 (+6 ms) 2024-11-13T05:37:08,830 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:37:08,834 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a47252b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:08,834 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:37:08,835 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:37:08,835 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:37:08,835 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:37:08,835 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:37:08,836 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:37:08,836 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:37:08,839 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:37:08,839 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:37:08,917 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:37:08,917 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:37:08,918 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:37:09,012 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:37:09,013 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:37:09,016 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:37:09,191 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:37:09,193 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:37:09,201 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:37:09,204 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:37:09,212 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:37:09,222 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:09,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:09,222 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,223 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,45157,1731476228451, sessionid=0x10132b284fc0000, setting cluster-up flag (Was=false) 2024-11-13T05:37:09,244 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,275 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:37:09,277 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,45157,1731476228451 2024-11-13T05:37:09,296 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,328 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:37:09,329 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,45157,1731476228451 2024-11-13T05:37:09,331 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:37:09,333 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:09,333 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:37:09,333 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:37:09,334 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,45157,1731476228451 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:37:09,335 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:09,335 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:09,336 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:09,336 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:09,336 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:37:09,336 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,336 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:37:09,336 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476259339 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:37:09,339 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:37:09,340 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,340 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:37:09,340 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:37:09,340 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:37:09,340 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:09,340 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:37:09,341 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:37:09,341 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:37:09,342 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:09,342 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476229342,5,FailOnTimeoutGroup] 2024-11-13T05:37:09,342 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476229342,5,FailOnTimeoutGroup] 2024-11-13T05:37:09,342 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,342 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:37:09,342 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:37:09,342 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,342 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:37:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:37:09,350 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:37:09,350 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817 2024-11-13T05:37:09,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:09,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:09,358 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:09,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:37:09,361 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:37:09,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:09,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:09,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:37:09,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:37:09,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:09,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:09,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:37:09,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:37:09,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:09,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:09,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:37:09,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:37:09,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:09,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:09,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:37:09,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740 2024-11-13T05:37:09,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740 2024-11-13T05:37:09,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:37:09,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:37:09,373 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:37:09,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:37:09,376 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:09,376 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876718, jitterRate=0.11480502784252167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:37:09,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476229358Initializing all the Stores at 1731476229359 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476229359Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476229360 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476229360Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476229360Cleaning up temporary data from old regions at 1731476229372 (+12 ms)Region opened successfully at 1731476229377 (+5 ms) 2024-11-13T05:37:09,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:37:09,377 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:37:09,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:37:09,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:37:09,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:37:09,378 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:09,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476229377Disabling compacts and flushes for region at 1731476229377Disabling writes for close at 1731476229377Writing region close event to WAL at 1731476229378 (+1 ms)Closed at 1731476229378 2024-11-13T05:37:09,379 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:09,379 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:37:09,380 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(746): ClusterId : 107b19c6-4e72-447a-a76d-be515e236e6a 2024-11-13T05:37:09,380 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:37:09,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:37:09,381 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:37:09,383 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:37:09,425 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:37:09,425 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:37:09,435 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:37:09,436 DEBUG [RS:0;a61c20bd729c:44475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239bd6f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:09,453 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:44475 2024-11-13T05:37:09,453 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:37:09,453 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:37:09,453 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:37:09,454 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,45157,1731476228451 with port=44475, startcode=1731476228646 2024-11-13T05:37:09,454 DEBUG [RS:0;a61c20bd729c:44475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:37:09,456 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:37:09,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45157 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45157 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,458 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817 2024-11-13T05:37:09,458 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41149 2024-11-13T05:37:09,458 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:37:09,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:09,471 DEBUG [RS:0;a61c20bd729c:44475 {}] zookeeper.ZKUtil(111): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,471 WARN [RS:0;a61c20bd729c:44475 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:37:09,471 INFO [RS:0;a61c20bd729c:44475 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:09,472 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,44475,1731476228646] 2024-11-13T05:37:09,476 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:37:09,479 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:37:09,480 INFO [RS:0;a61c20bd729c:44475 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:37:09,480 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,480 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:37:09,481 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:37:09,481 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,482 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,483 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,483 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,483 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,483 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:09,483 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:37:09,483 DEBUG [RS:0;a61c20bd729c:44475 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:37:09,484 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,484 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,484 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,484 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,484 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,484 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,44475,1731476228646-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:37:09,498 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:37:09,498 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,44475,1731476228646-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,498 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,498 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.Replication(171): a61c20bd729c,44475,1731476228646 started 2024-11-13T05:37:09,513 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:09,513 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,44475,1731476228646, RpcServer on a61c20bd729c/172.17.0.3:44475, sessionid=0x10132b284fc0001 2024-11-13T05:37:09,513 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:37:09,513 DEBUG [RS:0;a61c20bd729c:44475 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,513 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,44475,1731476228646' 2024-11-13T05:37:09,513 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:37:09,514 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:37:09,515 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:37:09,515 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:37:09,515 DEBUG [RS:0;a61c20bd729c:44475 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,515 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,44475,1731476228646' 2024-11-13T05:37:09,515 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:37:09,515 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:37:09,516 DEBUG [RS:0;a61c20bd729c:44475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:37:09,516 INFO [RS:0;a61c20bd729c:44475 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:37:09,516 INFO [RS:0;a61c20bd729c:44475 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:37:09,533 WARN [a61c20bd729c:45157 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:37:09,621 INFO [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C44475%2C1731476228646, suffix=, logDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646, archiveDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs, maxLogs=32 2024-11-13T05:37:09,623 INFO [RS:0;a61c20bd729c:44475 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.1731476229623 2024-11-13T05:37:09,631 INFO [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 2024-11-13T05:37:09,632 DEBUG [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45421:45421),(127.0.0.1/127.0.0.1:40671:40671)] 2024-11-13T05:37:09,783 DEBUG [a61c20bd729c:45157 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:37:09,784 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,44475,1731476228646 2024-11-13T05:37:09,786 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,44475,1731476228646, state=OPENING 2024-11-13T05:37:09,840 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:37:09,850 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:09,851 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:37:09,851 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:09,851 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:09,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,44475,1731476228646}] 2024-11-13T05:37:10,009 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:37:10,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:37:10,043 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:37:10,043 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:10,045 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C44475%2C1731476228646.meta, suffix=.meta, logDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646, archiveDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs, maxLogs=32 2024-11-13T05:37:10,046 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta 2024-11-13T05:37:10,051 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta 2024-11-13T05:37:10,052 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40671:40671),(127.0.0.1/127.0.0.1:45421:45421)] 2024-11-13T05:37:10,052 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:37:10,053 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:37:10,053 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:37:10,053 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:37:10,053 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:37:10,053 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:10,053 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:37:10,053 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:37:10,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:37:10,055 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:37:10,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:10,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:10,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:37:10,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:37:10,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:10,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:10,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:37:10,058 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:37:10,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:10,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:10,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:37:10,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:37:10,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:10,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:10,060 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:37:10,061 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740 2024-11-13T05:37:10,062 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740 2024-11-13T05:37:10,063 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:37:10,063 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:37:10,064 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:37:10,065 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:37:10,066 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874958, jitterRate=0.11256703734397888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:37:10,066 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:37:10,067 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476230053Writing region info on filesystem at 1731476230053Initializing all the Stores at 1731476230054 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476230054Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476230054Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476230054Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476230054Cleaning up temporary data from old regions at 1731476230063 (+9 ms)Running coprocessor post-open hooks at 1731476230066 (+3 ms)Region opened successfully at 1731476230066 2024-11-13T05:37:10,068 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476230008 2024-11-13T05:37:10,070 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:37:10,070 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:37:10,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,44475,1731476228646 2024-11-13T05:37:10,072 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,44475,1731476228646, state=OPEN 2024-11-13T05:37:10,133 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:37:10,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:37:10,133 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,44475,1731476228646 2024-11-13T05:37:10,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:10,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:37:10,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:37:10,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,44475,1731476228646 in 282 msec 2024-11-13T05:37:10,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:37:10,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 756 msec 2024-11-13T05:37:10,140 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:10,140 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:37:10,142 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:37:10,142 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,44475,1731476228646, seqNum=-1] 2024-11-13T05:37:10,142 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:37:10,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:37:10,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 817 msec 2024-11-13T05:37:10,150 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476230150, completionTime=-1 2024-11-13T05:37:10,150 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:37:10,151 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:37:10,152 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476290153 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476350153 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,45157,1731476228451-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,45157,1731476228451-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,45157,1731476228451-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:45157, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,153 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,154 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,155 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.434sec 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,45157,1731476228451-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:37:10,158 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,45157,1731476228451-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:37:10,161 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:37:10,161 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:37:10,161 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,45157,1731476228451-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c67fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:37:10,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,45157,-1 for getting cluster id 2024-11-13T05:37:10,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:37:10,183 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '107b19c6-4e72-447a-a76d-be515e236e6a' 2024-11-13T05:37:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:37:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "107b19c6-4e72-447a-a76d-be515e236e6a" 2024-11-13T05:37:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a78b050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:37:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,45157,-1] 2024-11-13T05:37:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:37:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:10,186 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:37:10,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365affa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:37:10,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:37:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,44475,1731476228646, seqNum=-1] 2024-11-13T05:37:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:37:10,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:37:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,45157,1731476228451 2024-11-13T05:37:10,194 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:10,196 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:37:10,213 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:37:10,213 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:10,214 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35337 2024-11-13T05:37:10,215 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35337 connecting to ZooKeeper ensemble=127.0.0.1:51491 2024-11-13T05:37:10,216 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:10,217 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:10,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353370x0, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:10,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:353370x0, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-13T05:37:10,240 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-13T05:37:10,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35337-0x10132b284fc0002 connected 2024-11-13T05:37:10,240 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:37:10,241 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:37:10,241 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:37:10,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:10,246 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35337 2024-11-13T05:37:10,246 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35337 2024-11-13T05:37:10,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35337 2024-11-13T05:37:10,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35337 2024-11-13T05:37:10,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35337 2024-11-13T05:37:10,250 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(746): ClusterId : 107b19c6-4e72-447a-a76d-be515e236e6a 2024-11-13T05:37:10,250 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:37:10,261 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:37:10,261 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:37:10,272 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:37:10,273 DEBUG [RS:1;a61c20bd729c:35337 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c16780c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:10,290 DEBUG [RS:1;a61c20bd729c:35337 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a61c20bd729c:35337 2024-11-13T05:37:10,290 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:37:10,290 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:37:10,290 DEBUG [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:37:10,291 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,45157,1731476228451 with port=35337, startcode=1731476230212 2024-11-13T05:37:10,291 DEBUG [RS:1;a61c20bd729c:35337 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:37:10,293 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55175, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:37:10,293 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45157 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,35337,1731476230212 2024-11-13T05:37:10,294 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45157 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,35337,1731476230212 2024-11-13T05:37:10,295 DEBUG [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817 2024-11-13T05:37:10,295 DEBUG [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41149 2024-11-13T05:37:10,295 DEBUG [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:37:10,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:10,307 DEBUG [RS:1;a61c20bd729c:35337 {}] zookeeper.ZKUtil(111): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,35337,1731476230212 2024-11-13T05:37:10,307 WARN [RS:1;a61c20bd729c:35337 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:37:10,307 INFO [RS:1;a61c20bd729c:35337 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:10,307 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,35337,1731476230212] 2024-11-13T05:37:10,307 DEBUG [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212 2024-11-13T05:37:10,312 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:37:10,315 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:37:10,315 INFO [RS:1;a61c20bd729c:35337 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:37:10,315 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,315 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:37:10,316 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:37:10,317 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,317 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,318 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,318 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:10,318 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:37:10,318 DEBUG [RS:1;a61c20bd729c:35337 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:37:10,318 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,318 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,318 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,318 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,318 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,319 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35337,1731476230212-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:37:10,333 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:37:10,333 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35337,1731476230212-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,333 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,334 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.Replication(171): a61c20bd729c,35337,1731476230212 started 2024-11-13T05:37:10,346 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:10,346 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,35337,1731476230212, RpcServer on a61c20bd729c/172.17.0.3:35337, sessionid=0x10132b284fc0002 2024-11-13T05:37:10,346 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:37:10,346 DEBUG [RS:1;a61c20bd729c:35337 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,35337,1731476230212 2024-11-13T05:37:10,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;a61c20bd729c:35337,5,FailOnTimeoutGroup] 2024-11-13T05:37:10,346 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,35337,1731476230212' 2024-11-13T05:37:10,346 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:37:10,346 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-11-13T05:37:10,347 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:37:10,347 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:37:10,347 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:37:10,347 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:37:10,347 DEBUG [RS:1;a61c20bd729c:35337 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,35337,1731476230212 2024-11-13T05:37:10,347 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,35337,1731476230212' 2024-11-13T05:37:10,347 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:37:10,348 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:37:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is a61c20bd729c,45157,1731476228451 2024-11-13T05:37:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ec62ed4 2024-11-13T05:37:10,348 DEBUG [RS:1;a61c20bd729c:35337 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:37:10,348 INFO [RS:1;a61c20bd729c:35337 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:37:10,348 INFO [RS:1;a61c20bd729c:35337 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:37:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:37:10,350 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:37:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-13T05:37:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-13T05:37:10,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:37:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-13T05:37:10,354 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:37:10,354 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:10,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-13T05:37:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:37:10,355 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:37:10,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741835_1011 (size=393) 2024-11-13T05:37:10,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741835_1011 (size=393) 2024-11-13T05:37:10,364 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d6a78b71d26d2d3175aa6971b92f55d4, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817 2024-11-13T05:37:10,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36475 is added to blk_1073741836_1012 (size=76) 2024-11-13T05:37:10,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33291 is added to blk_1073741836_1012 (size=76) 2024-11-13T05:37:10,370 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:10,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing d6a78b71d26d2d3175aa6971b92f55d4, disabling compactions & flushes 2024-11-13T05:37:10,371 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. after waiting 0 ms 2024-11-13T05:37:10,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,371 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,371 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for d6a78b71d26d2d3175aa6971b92f55d4: Waiting for close lock at 1731476230370Disabling compacts and flushes for region at 1731476230370Disabling writes for close at 1731476230371 (+1 ms)Writing region close event to WAL at 1731476230371Closed at 1731476230371 2024-11-13T05:37:10,372 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:37:10,373 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731476230373"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476230373"}]},"ts":"1731476230373"} 2024-11-13T05:37:10,375 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:37:10,376 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:37:10,377 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476230377"}]},"ts":"1731476230377"} 2024-11-13T05:37:10,379 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-13T05:37:10,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d6a78b71d26d2d3175aa6971b92f55d4, ASSIGN}] 2024-11-13T05:37:10,381 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d6a78b71d26d2d3175aa6971b92f55d4, ASSIGN 2024-11-13T05:37:10,383 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d6a78b71d26d2d3175aa6971b92f55d4, ASSIGN; state=OFFLINE, location=a61c20bd729c,44475,1731476228646; forceNewPlan=false, retain=false 2024-11-13T05:37:10,451 INFO [RS:1;a61c20bd729c:35337 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C35337%2C1731476230212, suffix=, logDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212, archiveDir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs, maxLogs=32 2024-11-13T05:37:10,452 INFO [RS:1;a61c20bd729c:35337 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35337%2C1731476230212.1731476230452 2024-11-13T05:37:10,460 INFO [RS:1;a61c20bd729c:35337 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 2024-11-13T05:37:10,464 DEBUG [RS:1;a61c20bd729c:35337 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40671:40671),(127.0.0.1/127.0.0.1:45421:45421)] 2024-11-13T05:37:10,534 INFO [a61c20bd729c:45157 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:37:10,535 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6a78b71d26d2d3175aa6971b92f55d4, regionState=OPENING, regionLocation=a61c20bd729c,44475,1731476228646 2024-11-13T05:37:10,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d6a78b71d26d2d3175aa6971b92f55d4, ASSIGN because future has completed 2024-11-13T05:37:10,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6a78b71d26d2d3175aa6971b92f55d4, server=a61c20bd729c,44475,1731476228646}] 2024-11-13T05:37:10,706 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,706 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d6a78b71d26d2d3175aa6971b92f55d4, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:37:10,707 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,707 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:10,707 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,707 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,710 INFO [StoreOpener-d6a78b71d26d2d3175aa6971b92f55d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,712 INFO [StoreOpener-d6a78b71d26d2d3175aa6971b92f55d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6a78b71d26d2d3175aa6971b92f55d4 columnFamilyName info 2024-11-13T05:37:10,712 DEBUG [StoreOpener-d6a78b71d26d2d3175aa6971b92f55d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:10,713 INFO [StoreOpener-d6a78b71d26d2d3175aa6971b92f55d4-1 {}] regionserver.HStore(327): Store=d6a78b71d26d2d3175aa6971b92f55d4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:10,713 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,714 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,714 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,715 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,715 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,717 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,719 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:10,720 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d6a78b71d26d2d3175aa6971b92f55d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728765, jitterRate=-0.0733279436826706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:37:10,720 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:10,721 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d6a78b71d26d2d3175aa6971b92f55d4: Running coprocessor pre-open hook at 1731476230708Writing region info on filesystem at 1731476230708Initializing all the Stores at 1731476230709 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476230709Cleaning up temporary data from old regions at 1731476230715 (+6 ms)Running coprocessor post-open hooks at 1731476230720 (+5 ms)Region opened successfully at 1731476230721 (+1 ms) 2024-11-13T05:37:10,722 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4., pid=6, masterSystemTime=1731476230697 2024-11-13T05:37:10,725 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,726 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:10,727 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6a78b71d26d2d3175aa6971b92f55d4, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,44475,1731476228646 2024-11-13T05:37:10,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6a78b71d26d2d3175aa6971b92f55d4, server=a61c20bd729c,44475,1731476228646 because future has completed 2024-11-13T05:37:10,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:37:10,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d6a78b71d26d2d3175aa6971b92f55d4, server=a61c20bd729c,44475,1731476228646 in 191 msec 2024-11-13T05:37:10,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:37:10,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d6a78b71d26d2d3175aa6971b92f55d4, ASSIGN in 357 msec 2024-11-13T05:37:10,740 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:37:10,741 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476230741"}]},"ts":"1731476230741"} 2024-11-13T05:37:10,744 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-13T05:37:10,745 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:37:10,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 394 msec 2024-11-13T05:37:14,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:37:14,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:37:14,973 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-13T05:37:14,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-13T05:37:14,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:37:14,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:37:15,560 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:37:15,564 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:15,583 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:15,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:15,586 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:15,594 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-13T05:37:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:37:20,435 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-13T05:37:20,435 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-13T05:37:20,442 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-13T05:37:20,442 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:20,456 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:20,459 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:20,460 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:20,460 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:20,461 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:37:20,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a2ad893{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:20,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d01e731{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:20,554 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@580ed5d9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-39775-hadoop-hdfs-3_4_1-tests_jar-_-any-11521575289080661362/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:20,555 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7645500{HTTP/1.1, (http/1.1)}{localhost:39775} 2024-11-13T05:37:20,555 INFO [Time-limited test {}] server.Server(415): Started @122523ms 2024-11-13T05:37:20,556 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:20,584 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:20,587 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:20,588 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:20,588 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:20,588 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:37:20,589 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e1b0a79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:20,589 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c2aec0f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:20,686 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@13f15eee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-44287-hadoop-hdfs-3_4_1-tests_jar-_-any-17483895386267819691/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:20,686 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c4a6f04{HTTP/1.1, (http/1.1)}{localhost:44287} 2024-11-13T05:37:20,686 INFO [Time-limited test {}] server.Server(415): Started @122654ms 2024-11-13T05:37:20,687 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:20,721 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:20,724 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:20,725 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:20,725 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:20,725 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:20,725 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@306f846b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:20,725 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fde87bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:20,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@53e17b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-40631-hadoop-hdfs-3_4_1-tests_jar-_-any-3456631797203524620/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:20,857 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@50128429{HTTP/1.1, (http/1.1)}{localhost:40631} 2024-11-13T05:37:20,857 INFO [Time-limited test {}] server.Server(415): Started @122824ms 2024-11-13T05:37:20,858 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:22,050 WARN [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:22,050 WARN [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:22,065 WARN [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:22,071 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd6efd5b299e4d579 with lease ID 0xbb59ec497afcae0e: Processing first storage report for DS-395f2c32-5c20-4884-9420-efa4db5a2b5b from datanode DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:22,072 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6efd5b299e4d579 with lease ID 0xbb59ec497afcae0e: from storage DS-395f2c32-5c20-4884-9420-efa4db5a2b5b node DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:22,072 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd6efd5b299e4d579 with lease ID 0xbb59ec497afcae0e: Processing first storage report for DS-40eadbee-206c-4d6c-ae5b-45d843377692 from datanode DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:22,072 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6efd5b299e4d579 with lease ID 0xbb59ec497afcae0e: from storage DS-40eadbee-206c-4d6c-ae5b-45d843377692 node DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:22,296 WARN [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data7/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:22,298 WARN [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data8/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:22,320 WARN [Thread-831 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x35e883222e0789f0 with lease ID 0xbb59ec497afcae0f: Processing first storage report for DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31 from datanode DatanodeRegistration(127.0.0.1:37471, datanodeUuid=c5859acf-e457-423c-ab48-5e3e447f1932, infoPort=46399, infoSecurePort=0, ipcPort=33975, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x35e883222e0789f0 with lease ID 0xbb59ec497afcae0f: from storage DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31 node DatanodeRegistration(127.0.0.1:37471, datanodeUuid=c5859acf-e457-423c-ab48-5e3e447f1932, infoPort=46399, infoSecurePort=0, ipcPort=33975, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x35e883222e0789f0 with lease ID 0xbb59ec497afcae0f: Processing first storage report for DS-696068d1-2d0e-43ad-9741-080fd2b4e13c from datanode DatanodeRegistration(127.0.0.1:37471, datanodeUuid=c5859acf-e457-423c-ab48-5e3e447f1932, infoPort=46399, infoSecurePort=0, ipcPort=33975, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x35e883222e0789f0 with lease ID 0xbb59ec497afcae0f: from storage DS-696068d1-2d0e-43ad-9741-080fd2b4e13c node DatanodeRegistration(127.0.0.1:37471, datanodeUuid=c5859acf-e457-423c-ab48-5e3e447f1932, infoPort=46399, infoSecurePort=0, ipcPort=33975, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:22,423 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data9/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:22,428 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data10/current/BP-139784409-172.17.0.3-1731476226050/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:22,467 WARN [Thread-853 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:22,473 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a2fb41dac8dca60 with lease ID 0xbb59ec497afcae10: Processing first storage report for DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b from datanode DatanodeRegistration(127.0.0.1:34231, datanodeUuid=85badd3e-d4e3-4f2d-821c-2665ab74f01b, infoPort=44495, infoSecurePort=0, ipcPort=39869, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:22,473 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a2fb41dac8dca60 with lease ID 0xbb59ec497afcae10: from storage DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b node DatanodeRegistration(127.0.0.1:34231, datanodeUuid=85badd3e-d4e3-4f2d-821c-2665ab74f01b, infoPort=44495, infoSecurePort=0, ipcPort=39869, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:22,473 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a2fb41dac8dca60 with lease ID 0xbb59ec497afcae10: Processing first storage report for DS-b5763f80-21b3-45d6-b9f2-b4a6af9ab8f2 from datanode DatanodeRegistration(127.0.0.1:34231, datanodeUuid=85badd3e-d4e3-4f2d-821c-2665ab74f01b, infoPort=44495, infoSecurePort=0, ipcPort=39869, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050) 2024-11-13T05:37:22,473 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a2fb41dac8dca60 with lease ID 0xbb59ec497afcae10: from storage DS-b5763f80-21b3-45d6-b9f2-b4a6af9ab8f2 node DatanodeRegistration(127.0.0.1:34231, datanodeUuid=85badd3e-d4e3-4f2d-821c-2665ab74f01b, infoPort=44495, infoSecurePort=0, ipcPort=39869, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:22,494 WARN [ResponseProcessor for block BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,495 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 block BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK], DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:22,495 WARN [ResponseProcessor for block BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,496 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 block BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:22,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:60738 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36475:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60738 dst: /127.0.0.1:36475 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:42880 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42880 dst: /127.0.0.1:33291 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-606947386_22 at /127.0.0.1:42848 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42848 dst: /127.0.0.1:33291 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-606947386_22 at /127.0.0.1:60726 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36475:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60726 dst: /127.0.0.1:36475 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,500 WARN [ResponseProcessor for block BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013 java.io.IOException: Bad response ERROR for BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,501 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 block BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:22,501 WARN [PacketResponder: BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33291] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,501 WARN [ResponseProcessor for block BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,501 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1250001090_22 at /127.0.0.1:60780 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:36475:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60780 dst: /127.0.0.1:36475 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1250001090_22 at /127.0.0.1:42928 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:33291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42928 dst: /127.0.0.1:33291 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,504 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta block BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:22,504 WARN [PacketResponder: BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33291] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:60752 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36475:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60752 dst: /127.0.0.1:36475 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:42892 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42892 dst: /127.0.0.1:33291 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,565 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6cc9fc90{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:22,565 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7909691f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:22,565 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:22,566 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d227195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:22,566 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f186434{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:22,567 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:22,567 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:22,567 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-139784409-172.17.0.3-1731476226050 (Datanode Uuid e9c7d410-ca24-42f8-8b74-27a3740d53c6) service to localhost/127.0.0.1:41149 2024-11-13T05:37:22,567 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:22,568 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data3/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:22,568 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data4/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:22,568 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:22,574 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 block BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,581 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@cb9f349 {}] datanode.DataXceiver(331): 127.0.0.1:36475:DataXceiver error processing unknown operation src: /127.0.0.1:53496 dst: /127.0.0.1:36475 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:22,582 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 block BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,582 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 block BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,583 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta block BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,583 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@527a79d3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:22,584 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78ac40e9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:22,584 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:22,584 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ed9c958{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:22,584 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ad2b72e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:22,589 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:22,589 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:22,589 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-139784409-172.17.0.3-1731476226050 (Datanode Uuid ae4362a4-1e1a-4b20-aee5-e9c92751a9a0) service to localhost/127.0.0.1:41149 2024-11-13T05:37:22,589 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:22,590 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data1/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:22,590 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data2/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:22,590 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4., hostname=a61c20bd729c,44475,1731476228646, seqNum=2] 2024-11-13T05:37:22,596 ERROR [FSHLog-0-hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817-prefix:a61c20bd729c,44475,1731476228646 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,596 WARN [FSHLog-0-hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817-prefix:a61c20bd729c,44475,1731476228646 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,596 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,596 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C44475%2C1731476228646:(num 1731476229623) roll requested 2024-11-13T05:37:22,597 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.1731476242597 2024-11-13T05:37:22,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:22,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:22,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:22,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:22,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:22,603 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476242597 2024-11-13T05:37:22,604 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,604 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:22,605 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T05:37:22,605 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T05:37:22,605 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 2024-11-13T05:37:22,608 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46399:46399),(127.0.0.1/127.0.0.1:44495:44495)] 2024-11-13T05:37:22,608 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:22,609 WARN [IPC Server handler 3 on default port 41149 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009 2024-11-13T05:37:22,614 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 after 6ms 2024-11-13T05:37:22,776 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:24,319 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:24,608 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:24,610 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476242597 2024-11-13T05:37:24,610 WARN [ResponseProcessor for block BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:24,611 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476242597 block BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:24,611 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:58068 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:37471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58068 dst: /127.0.0.1:37471 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:24,611 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:55868 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:34231:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55868 dst: /127.0.0.1:34231 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:24,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@13f15eee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:24,700 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c4a6f04{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:24,701 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:24,701 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c2aec0f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:24,701 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e1b0a79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:24,702 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:24,702 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:24,702 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:24,702 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-139784409-172.17.0.3-1731476226050 (Datanode Uuid c5859acf-e457-423c-ab48-5e3e447f1932) service to localhost/127.0.0.1:41149 2024-11-13T05:37:24,702 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data7/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:24,702 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data8/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:24,703 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:24,777 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:26,319 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:26,609 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:26,609 WARN [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]] 2024-11-13T05:37:26,609 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C44475%2C1731476228646:(num 1731476242597) roll requested 2024-11-13T05:37:26,610 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.1731476246610 2024-11-13T05:37:26,614 WARN [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:26,614 WARN [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:26,614 WARN [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741839_1021 2024-11-13T05:37:26,615 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 after 4009ms 2024-11-13T05:37:26,617 WARN [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:26,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:26,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:26,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:26,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:26,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:26,625 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476242597 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476246610 2024-11-13T05:37:26,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34231 is added to blk_1073741838_1020 (size=2431) 2024-11-13T05:37:26,634 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40861:40861),(127.0.0.1/127.0.0.1:44495:44495)] 2024-11-13T05:37:26,634 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:26,634 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476242597 is not closed yet, will try archiving it next time 2024-11-13T05:37:26,707 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:37:26,777 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:27,028 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:28,320 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,534 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@569027fc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34231, datanodeUuid=85badd3e-d4e3-4f2d-821c-2665ab74f01b, infoPort=44495, infoSecurePort=0, ipcPort=39869, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741838_1020 to 127.0.0.1:36475 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:28,635 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,711 WARN [ResponseProcessor for block BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022 java.io.IOException: Bad response ERROR for BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022 from datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,712 WARN [DataStreamer for file /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476246610 block BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:28,712 WARN [PacketResponder: BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:34231] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:28,712 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:48982 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48982 dst: /127.0.0.1:40609 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:28,712 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:55878 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:34231:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55878 dst: /127.0.0.1:34231 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:28,757 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@53e17b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:28,757 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@50128429{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:28,757 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:28,757 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fde87bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:28,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@306f846b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:28,759 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:28,759 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-139784409-172.17.0.3-1731476226050 (Datanode Uuid 85badd3e-d4e3-4f2d-821c-2665ab74f01b) service to localhost/127.0.0.1:41149 2024-11-13T05:37:28,759 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:28,759 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:28,759 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data9/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:28,759 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data10/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:28,760 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44475 {}] regionserver.HRegion(8855): Flush requested on d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:28,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:37:28,777 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/35b22083350c442fac98618a66ebdaff is 1080, key is row0002/info:/1731476244704/Put/seqid=0 2024-11-13T05:37:28,791 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,791 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:28,791 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741841_1024 2024-11-13T05:37:28,791 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:28,793 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,793 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:28,793 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741842_1025 2024-11-13T05:37:28,794 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:28,795 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,795 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:28,795 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741843_1026 2024-11-13T05:37:28,796 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:28,799 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33291 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:28,799 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:28,799 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:48992 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741844_1027 to mirror 127.0.0.1:33291 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:28,799 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741844_1027 2024-11-13T05:37:28,799 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:48992 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:28,799 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:48992 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48992 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:28,800 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:28,800 WARN [IPC Server handler 3 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:28,800 WARN [IPC Server handler 3 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:28,801 WARN [IPC Server handler 3 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741845_1028 (size=10347) 2024-11-13T05:37:29,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/35b22083350c442fac98618a66ebdaff 2024-11-13T05:37:29,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/35b22083350c442fac98618a66ebdaff as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/35b22083350c442fac98618a66ebdaff 2024-11-13T05:37:29,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/35b22083350c442fac98618a66ebdaff, entries=5, sequenceid=11, filesize=10.1 K 2024-11-13T05:37:29,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for d6a78b71d26d2d3175aa6971b92f55d4 in 451ms, sequenceid=11, compaction requested=false 2024-11-13T05:37:29,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44475 {}] regionserver.HRegion(8855): Flush requested on d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:29,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-11-13T05:37:29,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/958c9973c20f48a181bfc1b3f4b17cfa is 1080, key is row0007/info:/1731476248771/Put/seqid=0 2024-11-13T05:37:29,404 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:29,404 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:29,404 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741846_1029 2024-11-13T05:37:29,405 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:29,407 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:29,407 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:29,407 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741847_1030 2024-11-13T05:37:29,407 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:29,409 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:29,409 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:29,409 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741848_1031 2024-11-13T05:37:29,410 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:29,412 WARN [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34231 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:29,412 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49016 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741849_1032 to mirror 127.0.0.1:34231 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:29,412 WARN [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:29,412 WARN [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741849_1032 2024-11-13T05:37:29,412 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49016 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:29,412 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49016 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49016 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:29,413 WARN [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:29,414 WARN [IPC Server handler 2 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:29,414 WARN [IPC Server handler 2 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:29,414 WARN [IPC Server handler 2 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:29,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741850_1033 (size=12506) 2024-11-13T05:37:29,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/958c9973c20f48a181bfc1b3f4b17cfa 2024-11-13T05:37:29,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/958c9973c20f48a181bfc1b3f4b17cfa as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa 2024-11-13T05:37:29,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa, entries=7, sequenceid=24, filesize=12.2 K 2024-11-13T05:37:29,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for d6a78b71d26d2d3175aa6971b92f55d4 in 441ms, sequenceid=24, compaction requested=false 2024-11-13T05:37:29,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:29,839 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-11-13T05:37:29,839 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:29,839 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa because midkey is the same as first or last row 2024-11-13T05:37:30,320 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,635 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,635 WARN [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]] 2024-11-13T05:37:30,635 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C44475%2C1731476228646:(num 1731476246610) roll requested 2024-11-13T05:37:30,636 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.1731476250635 2024-11-13T05:37:30,639 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,639 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:30,639 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741851_1034 2024-11-13T05:37:30,640 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:30,641 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,641 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:30,641 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741852_1035 2024-11-13T05:37:30,642 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:30,643 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,643 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:30,643 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741853_1036 2024-11-13T05:37:30,644 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:30,646 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34231 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,646 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49030 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741854_1037 to mirror 127.0.0.1:34231 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,646 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:30,646 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49030 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-13T05:37:30,646 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741854_1037 2024-11-13T05:37:30,647 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49030 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49030 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,647 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:30,648 WARN [IPC Server handler 1 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:30,648 WARN [IPC Server handler 1 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:30,648 WARN [IPC Server handler 1 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:30,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:30,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:30,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:30,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:30,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:30,652 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476246610 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476250635 2024-11-13T05:37:30,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741840_1023 (size=25992) 2024-11-13T05:37:30,654 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40861:40861)] 2024-11-13T05:37:30,654 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:30,654 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476246610 is not closed yet, will try archiving it next time 2024-11-13T05:37:30,654 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476242597 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs/a61c20bd729c%2C44475%2C1731476228646.1731476242597 2024-11-13T05:37:30,778 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44475 {}] regionserver.HRegion(8855): Flush requested on d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:30,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-13T05:37:30,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/9bafc648a5d3402298938b3abd0cdd22 is 1079, key is tmprow/info:/1731476250816/Put/seqid=0 2024-11-13T05:37:30,825 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37471 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49056 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741856_1039 to mirror 127.0.0.1:37471 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,825 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:30,825 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741856_1039 2024-11-13T05:37:30,825 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49056 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:30,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49056 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49056 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,826 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:30,828 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34231 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,828 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49062 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741857_1040 to mirror 127.0.0.1:34231 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,828 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:30,828 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741857_1040 2024-11-13T05:37:30,828 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49062 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:30,828 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49062 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49062 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,829 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:30,831 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36475 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,831 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49070 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741858_1041 to mirror 127.0.0.1:36475 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,831 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:30,831 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741858_1041 2024-11-13T05:37:30,831 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49070 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:30,832 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49070 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49070 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,832 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:30,835 WARN [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33291 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:30,835 WARN [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:30,835 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49076 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741859_1042 to mirror 127.0.0.1:33291 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,835 WARN [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741859_1042 2024-11-13T05:37:30,835 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49076 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:30,835 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49076 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49076 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:30,836 WARN [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:30,836 WARN [IPC Server handler 0 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:30,836 WARN [IPC Server handler 0 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:30,837 WARN [IPC Server handler 0 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:30,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741860_1043 (size=6027) 2024-11-13T05:37:31,055 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:31,075 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3f14dabc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741850_1033 to 127.0.0.1:34231 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:31,075 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@31d332f6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741845_1028 to 127.0.0.1:33291 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:31,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/9bafc648a5d3402298938b3abd0cdd22 2024-11-13T05:37:31,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/9bafc648a5d3402298938b3abd0cdd22 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/9bafc648a5d3402298938b3abd0cdd22 2024-11-13T05:37:31,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/9bafc648a5d3402298938b3abd0cdd22, entries=1, sequenceid=34, filesize=5.9 K 2024-11-13T05:37:31,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for d6a78b71d26d2d3175aa6971b92f55d4 in 441ms, sequenceid=34, compaction requested=true 2024-11-13T05:37:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-11-13T05:37:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa because midkey is the same as first or last row 2024-11-13T05:37:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6a78b71d26d2d3175aa6971b92f55d4:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:37:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:37:31,259 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:37:31,261 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:37:31,261 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HStore(1541): d6a78b71d26d2d3175aa6971b92f55d4/info is initiating minor compaction (all files) 2024-11-13T05:37:31,261 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6a78b71d26d2d3175aa6971b92f55d4/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:31,261 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/35b22083350c442fac98618a66ebdaff, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/9bafc648a5d3402298938b3abd0cdd22] into tmpdir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp, totalSize=28.2 K 2024-11-13T05:37:31,261 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35b22083350c442fac98618a66ebdaff, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731476244704 2024-11-13T05:37:31,262 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.Compactor(225): Compacting 958c9973c20f48a181bfc1b3f4b17cfa, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731476248771 2024-11-13T05:37:31,262 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bafc648a5d3402298938b3abd0cdd22, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731476250816 2024-11-13T05:37:31,275 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6a78b71d26d2d3175aa6971b92f55d4#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:37:31,276 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/fcec793c205d4771ad521cac41b2981f is 1080, key is row0002/info:/1731476244704/Put/seqid=0 2024-11-13T05:37:31,278 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:31,278 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:31,278 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741861_1044 2024-11-13T05:37:31,279 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:31,281 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33291 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:31,281 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49108 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741862_1045 to mirror 127.0.0.1:33291 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:31,281 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:31,281 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49108 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:31,281 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741862_1045 2024-11-13T05:37:31,281 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49108 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49108 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:31,282 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:31,283 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:31,283 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:31,283 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741863_1046 2024-11-13T05:37:31,284 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:31,286 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36475 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:31,286 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49122 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741864_1047 to mirror 127.0.0.1:36475 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:31,287 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:31,287 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49122 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:31,287 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741864_1047 2024-11-13T05:37:31,287 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49122 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49122 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:31,287 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:31,288 WARN [IPC Server handler 1 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:31,288 WARN [IPC Server handler 1 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:31,288 WARN [IPC Server handler 1 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:31,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741865_1048 (size=17994) 2024-11-13T05:37:31,699 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/fcec793c205d4771ad521cac41b2981f as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f 2024-11-13T05:37:31,706 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6a78b71d26d2d3175aa6971b92f55d4/info of d6a78b71d26d2d3175aa6971b92f55d4 into fcec793c205d4771ad521cac41b2981f(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:37:31,706 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:31,706 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4., storeName=d6a78b71d26d2d3175aa6971b92f55d4/info, priority=13, startTime=1731476251259; duration=0sec 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f because midkey is the same as first or last row 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f because midkey is the same as first or last row 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f because midkey is the same as first or last row 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:37:31,707 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6a78b71d26d2d3175aa6971b92f55d4:info 2024-11-13T05:37:32,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3f14dabc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741840_1023 to 127.0.0.1:36475 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44475 {}] regionserver.HRegion(8855): Flush requested on d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:32,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-13T05:37:32,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/534a19f0b88849b382dc955e75df6f6f is 1079, key is tmprow/info:/1731476252236/Put/seqid=0 2024-11-13T05:37:32,243 WARN [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,243 WARN [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:32,243 WARN [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741866_1049 2024-11-13T05:37:32,244 WARN [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:32,245 WARN [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,245 WARN [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:32,245 WARN [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741867_1050 2024-11-13T05:37:32,246 WARN [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:32,248 WARN [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33291 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,248 WARN [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:32,248 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49934 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741868_1051 to mirror 127.0.0.1:33291 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,248 WARN [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741868_1051 2024-11-13T05:37:32,248 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49934 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:32,248 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49934 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49934 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,249 WARN [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:32,254 WARN [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37471 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,254 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49950 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741869_1052 to mirror 127.0.0.1:37471 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,254 WARN [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:32,254 WARN [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741869_1052 2024-11-13T05:37:32,254 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49950 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:32,254 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49950 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49950 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,255 WARN [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:32,255 WARN [IPC Server handler 0 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:32,255 WARN [IPC Server handler 0 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:32,256 WARN [IPC Server handler 0 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:32,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741870_1053 (size=6027) 2024-11-13T05:37:32,321 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,654 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,655 WARN [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]] 2024-11-13T05:37:32,655 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C44475%2C1731476228646:(num 1731476250635) roll requested 2024-11-13T05:37:32,655 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.1731476252655 2024-11-13T05:37:32,659 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36475 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,659 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49968 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741871_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741871_1054 to mirror 127.0.0.1:36475 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,659 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:32,659 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49968 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741871_1054] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-13T05:37:32,659 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741871_1054 2024-11-13T05:37:32,659 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49968 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741871_1054] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49968 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/534a19f0b88849b382dc955e75df6f6f 2024-11-13T05:37:32,660 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:32,661 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,662 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:32,662 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741872_1055 2024-11-13T05:37:32,662 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:32,664 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33291 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,664 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49980 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741873_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741873_1056 to mirror 127.0.0.1:33291 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,665 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:32,665 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49980 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741873_1056] {}] datanode.BlockReceiver(316): Block 1073741873 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-13T05:37:32,665 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741873_1056 2024-11-13T05:37:32,665 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49980 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741873_1056] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49980 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,665 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:32,668 WARN [Thread-962 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37471 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:32,667 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49992 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741874_1057 to mirror 127.0.0.1:37471 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,668 WARN [Thread-962 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:32,668 WARN [Thread-962 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741874_1057 2024-11-13T05:37:32,668 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49992 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-13T05:37:32,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:49992 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49992 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:32,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/534a19f0b88849b382dc955e75df6f6f as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/534a19f0b88849b382dc955e75df6f6f 2024-11-13T05:37:32,668 WARN [Thread-962 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:32,669 WARN [IPC Server handler 2 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:32,669 WARN [IPC Server handler 2 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:32,669 WARN [IPC Server handler 2 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:32,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:32,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:32,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:32,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:32,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:32,673 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476250635 with entries=14, filesize=12.92 KB; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476252655 2024-11-13T05:37:32,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741855_1038 (size=13234) 2024-11-13T05:37:32,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/534a19f0b88849b382dc955e75df6f6f, entries=1, sequenceid=45, filesize=5.9 K 2024-11-13T05:37:32,684 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40861:40861)] 2024-11-13T05:37:32,684 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:32,684 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476250635 is not closed yet, will try archiving it next time 2024-11-13T05:37:32,684 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476246610 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs/a61c20bd729c%2C44475%2C1731476228646.1731476246610 2024-11-13T05:37:32,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for d6a78b71d26d2d3175aa6971b92f55d4 in 448ms, sequenceid=45, compaction requested=false 2024-11-13T05:37:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-11-13T05:37:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:32,685 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f because midkey is the same as first or last row 2024-11-13T05:37:32,778 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:33,075 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 is not closed yet, will try archiving it next time 2024-11-13T05:37:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44475 {}] regionserver.HRegion(8855): Flush requested on d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:33,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-13T05:37:33,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/6f610cf61c4a4345bad8df6a4e0b137d is 1079, key is tmprow/info:/1731476253655/Put/seqid=0 2024-11-13T05:37:33,663 WARN [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:33,663 WARN [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:33,663 WARN [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741876_1059 2024-11-13T05:37:33,664 WARN [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:33,665 WARN [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:33,665 WARN [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:33,665 WARN [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741877_1060 2024-11-13T05:37:33,665 WARN [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:33,666 WARN [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:33,667 WARN [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:33,667 WARN [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741878_1061 2024-11-13T05:37:33,667 WARN [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:33,668 WARN [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:33,669 WARN [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:33,669 WARN [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741879_1062 2024-11-13T05:37:33,669 WARN [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:33,670 WARN [IPC Server handler 0 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:33,670 WARN [IPC Server handler 0 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:33,670 WARN [IPC Server handler 0 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:33,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741880_1063 (size=6027) 2024-11-13T05:37:34,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/6f610cf61c4a4345bad8df6a4e0b137d 2024-11-13T05:37:34,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@31d332f6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741865_1048 to 127.0.0.1:33291 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:34,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3f14dabc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741860_1043 to 127.0.0.1:36475 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:34,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/6f610cf61c4a4345bad8df6a4e0b137d as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/6f610cf61c4a4345bad8df6a4e0b137d 2024-11-13T05:37:34,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/6f610cf61c4a4345bad8df6a4e0b137d, entries=1, sequenceid=55, filesize=5.9 K 2024-11-13T05:37:34,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for d6a78b71d26d2d3175aa6971b92f55d4 in 433ms, sequenceid=55, compaction requested=true 2024-11-13T05:37:34,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:34,089 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K 2024-11-13T05:37:34,089 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:34,090 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f because midkey is the same as first or last row 2024-11-13T05:37:34,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6a78b71d26d2d3175aa6971b92f55d4:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:37:34,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:37:34,090 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:37:34,091 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:37:34,092 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HStore(1541): d6a78b71d26d2d3175aa6971b92f55d4/info is initiating minor compaction (all files) 2024-11-13T05:37:34,092 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6a78b71d26d2d3175aa6971b92f55d4/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:34,092 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/534a19f0b88849b382dc955e75df6f6f, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/6f610cf61c4a4345bad8df6a4e0b137d] into tmpdir=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp, totalSize=29.3 K 2024-11-13T05:37:34,093 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcec793c205d4771ad521cac41b2981f, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731476244704 2024-11-13T05:37:34,093 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534a19f0b88849b382dc955e75df6f6f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731476252236 2024-11-13T05:37:34,094 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f610cf61c4a4345bad8df6a4e0b137d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731476253655 2024-11-13T05:37:34,113 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6a78b71d26d2d3175aa6971b92f55d4#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:37:34,114 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/11a6c67be568411fa1e691f964cb0543 is 1080, key is row0002/info:/1731476244704/Put/seqid=0 2024-11-13T05:37:34,117 WARN [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33291 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:50016 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741881_1064] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741881_1064 to mirror 127.0.0.1:33291 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:34,117 WARN [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]) is bad. 2024-11-13T05:37:34,117 WARN [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741881_1064 2024-11-13T05:37:34,117 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:50016 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741881_1064] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:34,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:50016 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741881_1064] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50016 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:34,118 WARN [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33291,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK] 2024-11-13T05:37:34,119 WARN [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,119 WARN [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:34,119 WARN [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741882_1065 2024-11-13T05:37:34,120 WARN [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:34,121 WARN [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,122 WARN [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:34,122 WARN [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741883_1066 2024-11-13T05:37:34,122 WARN [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:34,124 WARN [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,124 WARN [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]) is bad. 2024-11-13T05:37:34,124 WARN [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741884_1067 2024-11-13T05:37:34,124 WARN [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK] 2024-11-13T05:37:34,125 WARN [IPC Server handler 3 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T05:37:34,125 WARN [IPC Server handler 3 on default port 41149 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T05:37:34,125 WARN [IPC Server handler 3 on default port 41149 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T05:37:34,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741885_1068 (size=18097) 2024-11-13T05:37:34,321 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,538 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/11a6c67be568411fa1e691f964cb0543 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/11a6c67be568411fa1e691f964cb0543 2024-11-13T05:37:34,546 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6a78b71d26d2d3175aa6971b92f55d4/info of d6a78b71d26d2d3175aa6971b92f55d4 into 11a6c67be568411fa1e691f964cb0543(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:34,546 INFO [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4., storeName=d6a78b71d26d2d3175aa6971b92f55d4/info, priority=13, startTime=1731476254090; duration=0sec 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/11a6c67be568411fa1e691f964cb0543 because midkey is the same as first or last row 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:34,546 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/11a6c67be568411fa1e691f964cb0543 because midkey is the same as first or last row 2024-11-13T05:37:34,547 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-13T05:37:34,547 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:34,547 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/11a6c67be568411fa1e691f964cb0543 because midkey is the same as first or last row 2024-11-13T05:37:34,547 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:37:34,547 DEBUG [RS:0;a61c20bd729c:44475-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6a78b71d26d2d3175aa6971b92f55d4:info 2024-11-13T05:37:34,685 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,685 WARN [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-11-13T05:37:34,778 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:34,880 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:34,884 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:34,885 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:34,885 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:34,885 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:34,886 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76c419ee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:34,886 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58dca5c7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:34,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@500e0366{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/java.io.tmpdir/jetty-localhost-36369-hadoop-hdfs-3_4_1-tests_jar-_-any-2598991145171587858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:34,985 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7900579e{HTTP/1.1, (http/1.1)}{localhost:36369} 2024-11-13T05:37:34,985 INFO [Time-limited test {}] server.Server(415): Started @136953ms 2024-11-13T05:37:34,986 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:35,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@31d332f6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741855_1038 to 127.0.0.1:36475 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:35,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3f14dabc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741870_1053 to 127.0.0.1:33291 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:35,388 WARN [Thread-993 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:35,397 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ab1796fa46714e9 with lease ID 0xbb59ec497afcae11: from storage DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f node DatanodeRegistration(127.0.0.1:45115, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45365, infoSecurePort=0, ipcPort=43351, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:35,397 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ab1796fa46714e9 with lease ID 0xbb59ec497afcae11: from storage DS-700af77e-f81e-4f89-90c1-372cdd8cffd5 node DatanodeRegistration(127.0.0.1:45115, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45365, infoSecurePort=0, ipcPort=43351, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:36,321 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:36,685 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:36,779 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:37,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@31d332f6[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741880_1063 to 127.0.0.1:34231 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:37,074 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3f14dabc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40609, datanodeUuid=efbb409e-f0e5-4411-a663-f71f252b3629, infoPort=40861, infoSecurePort=0, ipcPort=39227, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741885_1068 to 127.0.0.1:34231 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:38,322 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:38,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:37:38,685 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:38,779 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,340 ERROR [FSHLog-0-hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData-prefix:a61c20bd729c,45157,1731476228451 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,341 WARN [FSHLog-0-hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData-prefix:a61c20bd729c,45157,1731476228451 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,341 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C45157%2C1731476228451:(num 1731476228799) roll requested 2024-11-13T05:37:39,341 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C45157%2C1731476228451.1731476259341 2024-11-13T05:37:39,344 WARN [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,344 WARN [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:39,344 WARN [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741886_1069 2024-11-13T05:37:39,345 WARN [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:39,346 WARN [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,346 WARN [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK], DatanodeInfoWithStorage[127.0.0.1:45115,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK]) is bad. 2024-11-13T05:37:39,346 WARN [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741887_1070 2024-11-13T05:37:39,347 WARN [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37471,DS-a9240c24-ba51-49ee-bc04-2bc1a3446a31,DISK] 2024-11-13T05:37:39,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:39,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:39,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:39,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:39,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:39,351 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476259341 2024-11-13T05:37:39,352 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,352 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:39,352 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 2024-11-13T05:37:39,352 WARN [IPC Server handler 4 on default port 41149 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006 2024-11-13T05:37:39,353 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 after 1ms 2024-11-13T05:37:39,353 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40861:40861),(127.0.0.1/127.0.0.1:45365:45365)] 2024-11-13T05:37:39,353 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 is not closed yet, will try archiving it next time 2024-11-13T05:37:40,322 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:40,686 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:42,322 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:42,686 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:43,354 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 after 4002ms 2024-11-13T05:37:44,323 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:44,686 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:45,414 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2c34512d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:36475,null,null]) java.net.ConnectException: Call From a61c20bd729c/172.17.0.3 to localhost:45147 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-13T05:37:45,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741833_1019 (size=455) 2024-11-13T05:37:45,632 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476229623 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs/a61c20bd729c%2C44475%2C1731476228646.1731476229623 2024-11-13T05:37:45,633 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476250635 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs/a61c20bd729c%2C44475%2C1731476228646.1731476250635 2024-11-13T05:37:46,323 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:46,391 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1d523e5c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45115, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45365, infoSecurePort=0, ipcPort=43351, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741833_1019 to 127.0.0.1:37471 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:46,687 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,323 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,515 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.1731476268515 2024-11-13T05:37:48,518 WARN [Thread-1023 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,518 WARN [Thread-1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:45115,DS-04c48cb3-e65c-43b8-a493-9a7f5c31983f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:48,518 WARN [Thread-1023 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741889_1073 2024-11-13T05:37:48,519 WARN [Thread-1023 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:48,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,524 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476252655 with entries=14, filesize=12.95 KB; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476268515 2024-11-13T05:37:48,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40861:40861),(127.0.0.1/127.0.0.1:45365:45365)] 2024-11-13T05:37:48,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476252655 is not closed yet, will try archiving it next time 2024-11-13T05:37:48,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741875_1058 (size=13268) 2024-11-13T05:37:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44475 {}] regionserver.HRegion(8855): Flush requested on d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:48,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-13T05:37:48,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/a4d671dbfc084671be04e73cf5b06db3 is 1080, key is row0013/info:/1731476268527/Put/seqid=0 2024-11-13T05:37:48,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741891_1075 (size=8190) 2024-11-13T05:37:48,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741891_1075 (size=8190) 2024-11-13T05:37:48,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/a4d671dbfc084671be04e73cf5b06db3 2024-11-13T05:37:48,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/a4d671dbfc084671be04e73cf5b06db3 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/a4d671dbfc084671be04e73cf5b06db3 2024-11-13T05:37:48,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/a4d671dbfc084671be04e73cf5b06db3, entries=3, sequenceid=66, filesize=8.0 K 2024-11-13T05:37:48,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for d6a78b71d26d2d3175aa6971b92f55d4 in 32ms, sequenceid=66, compaction requested=false 2024-11-13T05:37:48,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6a78b71d26d2d3175aa6971b92f55d4: 2024-11-13T05:37:48,563 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K 2024-11-13T05:37:48,563 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:37:48,563 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/11a6c67be568411fa1e691f964cb0543 because midkey is the same as first or last row 2024-11-13T05:37:48,687 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,687 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-11-13T05:37:48,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:37:48,756 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:37:48,756 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:48,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:48,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:48,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:37:48,756 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:37:48,757 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=420766239, stopped=false 2024-11-13T05:37:48,757 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,45157,1731476228451 2024-11-13T05:37:48,851 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:48,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:48,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:48,851 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:48,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:48,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:37:48,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:48,852 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:37:48,852 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:48,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:48,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:48,852 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,44475,1731476228646' ***** 2024-11-13T05:37:48,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:48,852 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:37:48,852 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,35337,1731476230212' ***** 2024-11-13T05:37:48,852 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:37:48,852 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:37:48,853 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:48,853 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:37:48,853 INFO [RS:0;a61c20bd729c:44475 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:37:48,853 INFO [RS:0;a61c20bd729c:44475 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:37:48,853 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(3091): Received CLOSE for d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:37:48,853 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,35337,1731476230212 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:48,853 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,44475,1731476228646 2024-11-13T05:37:48,853 INFO [RS:0;a61c20bd729c:44475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a61c20bd729c:35337. 2024-11-13T05:37:48,853 INFO [RS:0;a61c20bd729c:44475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:44475. 2024-11-13T05:37:48,853 DEBUG [RS:1;a61c20bd729c:35337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:48,853 DEBUG [RS:1;a61c20bd729c:35337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:48,853 DEBUG [RS:0;a61c20bd729c:44475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:37:48,853 DEBUG [RS:0;a61c20bd729c:44475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:48,853 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6a78b71d26d2d3175aa6971b92f55d4, disabling compactions & flushes 2024-11-13T05:37:48,853 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,35337,1731476230212; all regions closed. 2024-11-13T05:37:48,854 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:48,854 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:48,854 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:37:48,854 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. after waiting 0 ms 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:48,854 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:37:48,854 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d6a78b71d26d2d3175aa6971b92f55d4 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-11-13T05:37:48,854 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:37:48,854 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d6a78b71d26d2d3175aa6971b92f55d4=TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.} 2024-11-13T05:37:48,854 DEBUG [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d6a78b71d26d2d3175aa6971b92f55d4 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:37:48,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,854 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:37:48,854 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:37:48,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,855 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-13T05:37:48,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,855 ERROR [FSHLog-0-hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817-prefix:a61c20bd729c,44475,1731476228646.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,855 WARN [FSHLog-0-hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817-prefix:a61c20bd729c,44475,1731476228646.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,855 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C44475%2C1731476228646.meta:.meta(num 1731476230046) roll requested 2024-11-13T05:37:48,856 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C44475%2C1731476228646.meta.1731476268855.meta 2024-11-13T05:37:48,860 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,860 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,860 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 2024-11-13T05:37:48,861 WARN [IPC Server handler 2 on default port 41149 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741837_1013 2024-11-13T05:37:48,861 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 after 1ms 2024-11-13T05:37:48,862 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/b479492cd28b4e66a9f49db96db25ca1 is 1080, key is row0015/info:/1731476268532/Put/seqid=0 2024-11-13T05:37:48,863 WARN [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1077 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34231 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56102 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741892_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741892_1077 to mirror 127.0.0.1:34231 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:48,863 WARN [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741892_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:48,863 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56102 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741892_1077] {}] datanode.BlockReceiver(316): Block 1073741892 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-13T05:37:48,863 WARN [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741892_1077 2024-11-13T05:37:48,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56102 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741892_1077] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56102 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:48,864 WARN [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:48,864 WARN [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1078 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,865 WARN [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741893_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:48,865 WARN [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741893_1078 2024-11-13T05:37:48,865 WARN [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:48,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:48,869 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476268855.meta 2024-11-13T05:37:48,869 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,869 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36475,DS-4bcd2f39-663e-4062-bb70-46c5e2f81a58,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,869 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta 2024-11-13T05:37:48,869 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40861:40861)] 2024-11-13T05:37:48,869 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta is not closed yet, will try archiving it next time 2024-11-13T05:37:48,870 WARN [IPC Server handler 1 on default port 41149 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741834_1010 2024-11-13T05:37:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741895_1080 (size=14660) 2024-11-13T05:37:48,870 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta after 1ms 2024-11-13T05:37:48,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741895_1080 (size=14660) 2024-11-13T05:37:48,871 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/b479492cd28b4e66a9f49db96db25ca1 2024-11-13T05:37:48,878 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/.tmp/info/b479492cd28b4e66a9f49db96db25ca1 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/b479492cd28b4e66a9f49db96db25ca1 2024-11-13T05:37:48,885 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/b479492cd28b4e66a9f49db96db25ca1, entries=9, sequenceid=78, filesize=14.3 K 2024-11-13T05:37:48,886 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for d6a78b71d26d2d3175aa6971b92f55d4 in 32ms, sequenceid=78, compaction requested=true 2024-11-13T05:37:48,887 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/35b22083350c442fac98618a66ebdaff, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/9bafc648a5d3402298938b3abd0cdd22, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/534a19f0b88849b382dc955e75df6f6f, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/6f610cf61c4a4345bad8df6a4e0b137d] to archive 2024-11-13T05:37:48,887 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/info/9450fac1ba7f4e84b68f3e65322173d1 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4./info:regioninfo/1731476230727/Put/seqid=0 2024-11-13T05:37:48,888 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:37:48,890 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56138 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741896_1082] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741896_1082 to mirror 127.0.0.1:34231 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:48,890 WARN [Thread-1050 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1082 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34231 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,891 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/35b22083350c442fac98618a66ebdaff to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/35b22083350c442fac98618a66ebdaff 2024-11-13T05:37:48,891 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56138 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741896_1082] {}] datanode.BlockReceiver(316): Block 1073741896 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:48,891 WARN [Thread-1050 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741896_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:48,891 WARN [Thread-1050 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741896_1082 2024-11-13T05:37:48,891 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56138 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741896_1082] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56138 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:48,892 WARN [Thread-1050 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:48,894 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/958c9973c20f48a181bfc1b3f4b17cfa 2024-11-13T05:37:48,896 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/fcec793c205d4771ad521cac41b2981f 2024-11-13T05:37:48,898 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/9bafc648a5d3402298938b3abd0cdd22 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/9bafc648a5d3402298938b3abd0cdd22 2024-11-13T05:37:48,900 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/534a19f0b88849b382dc955e75df6f6f to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/534a19f0b88849b382dc955e75df6f6f 2024-11-13T05:37:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741897_1083 (size=7089) 2024-11-13T05:37:48,902 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/6f610cf61c4a4345bad8df6a4e0b137d to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/info/6f610cf61c4a4345bad8df6a4e0b137d 2024-11-13T05:37:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741897_1083 (size=7089) 2024-11-13T05:37:48,902 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/info/9450fac1ba7f4e84b68f3e65322173d1 2024-11-13T05:37:48,902 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a61c20bd729c:45157 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-13T05:37:48,903 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [35b22083350c442fac98618a66ebdaff=10347, 958c9973c20f48a181bfc1b3f4b17cfa=12506, fcec793c205d4771ad521cac41b2981f=17994, 9bafc648a5d3402298938b3abd0cdd22=6027, 534a19f0b88849b382dc955e75df6f6f=6027, 6f610cf61c4a4345bad8df6a4e0b137d=6027] 2024-11-13T05:37:48,910 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d6a78b71d26d2d3175aa6971b92f55d4/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1 2024-11-13T05:37:48,911 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:48,911 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6a78b71d26d2d3175aa6971b92f55d4: Waiting for close lock at 1731476268853Running coprocessor pre-close hooks at 1731476268853Disabling compacts and flushes for region at 1731476268853Disabling writes for close at 1731476268854 (+1 ms)Obtaining lock to block concurrent updates at 1731476268854Preparing flush snapshotting stores in d6a78b71d26d2d3175aa6971b92f55d4 at 1731476268854Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4., syncing WAL and waiting on mvcc, flushsize=dataSize=9683, getHeapSize=10608, getOffHeapSize=0, getCellsCount=9 at 1731476268854Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. at 1731476268855 (+1 ms)Flushing d6a78b71d26d2d3175aa6971b92f55d4/info: creating writer at 1731476268855Flushing d6a78b71d26d2d3175aa6971b92f55d4/info: appending metadata at 1731476268862 (+7 ms)Flushing d6a78b71d26d2d3175aa6971b92f55d4/info: closing flushed file at 1731476268862Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61869efb: reopening flushed file at 1731476268877 (+15 ms)Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for d6a78b71d26d2d3175aa6971b92f55d4 in 32ms, sequenceid=78, compaction requested=true at 1731476268886 (+9 ms)Writing region close event to WAL at 1731476268904 (+18 ms)Running coprocessor post-close hooks at 1731476268911 (+7 ms)Closed at 1731476268911 2024-11-13T05:37:48,912 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731476230350.d6a78b71d26d2d3175aa6971b92f55d4. 2024-11-13T05:37:48,927 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.1731476252655 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs/a61c20bd729c%2C44475%2C1731476228646.1731476252655 2024-11-13T05:37:48,932 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/ns/648d3f4c361e4c9a9d387f44b4a43c59 is 43, key is default/ns:d/1731476230144/Put/seqid=0 2024-11-13T05:37:48,934 WARN [Thread-1058 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1084 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,934 WARN [Thread-1058 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741898_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK], DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:48,934 WARN [Thread-1058 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741898_1084 2024-11-13T05:37:48,935 WARN [Thread-1058 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:48,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741899_1085 (size=5153) 2024-11-13T05:37:48,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741899_1085 (size=5153) 2024-11-13T05:37:48,941 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/ns/648d3f4c361e4c9a9d387f44b4a43c59 2024-11-13T05:37:48,965 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/table/35978d5ff29248b595145b22b0c2554d is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731476230741/Put/seqid=0 2024-11-13T05:37:48,968 WARN [Thread-1064 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34231 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:37:48,968 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56174 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741900_1086] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6]'}, localName='127.0.0.1:40609', datanodeUuid='efbb409e-f0e5-4411-a663-f71f252b3629', xmitsInProgress=0}:Exception transferring block BP-139784409-172.17.0.3-1731476226050:blk_1073741900_1086 to mirror 127.0.0.1:34231 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:48,968 WARN [Thread-1064 {}] hdfs.DataStreamer(1731): Error Recovery for BP-139784409-172.17.0.3-1731476226050:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40609,DS-395f2c32-5c20-4884-9420-efa4db5a2b5b,DISK], DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK]) is bad. 2024-11-13T05:37:48,968 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56174 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741900_1086] {}] datanode.BlockReceiver(316): Block 1073741900 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-13T05:37:48,968 WARN [Thread-1064 {}] hdfs.DataStreamer(1850): Abandoning BP-139784409-172.17.0.3-1731476226050:blk_1073741900_1086 2024-11-13T05:37:48,969 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302249229_22 at /127.0.0.1:56174 [Receiving block BP-139784409-172.17.0.3-1731476226050:blk_1073741900_1086] {}] datanode.DataXceiver(331): 127.0.0.1:40609:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56174 dst: /127.0.0.1:40609 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:48,969 WARN [Thread-1064 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34231,DS-5bcadbc1-5d3b-4575-b474-6a9aebab8d4b,DISK] 2024-11-13T05:37:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741901_1087 (size=5424) 2024-11-13T05:37:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741901_1087 (size=5424) 2024-11-13T05:37:48,975 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/table/35978d5ff29248b595145b22b0c2554d 2024-11-13T05:37:48,984 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/info/9450fac1ba7f4e84b68f3e65322173d1 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/info/9450fac1ba7f4e84b68f3e65322173d1 2024-11-13T05:37:48,992 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/info/9450fac1ba7f4e84b68f3e65322173d1, entries=10, sequenceid=11, filesize=6.9 K 2024-11-13T05:37:48,993 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/ns/648d3f4c361e4c9a9d387f44b4a43c59 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/ns/648d3f4c361e4c9a9d387f44b4a43c59 2024-11-13T05:37:49,000 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/ns/648d3f4c361e4c9a9d387f44b4a43c59, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T05:37:49,001 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/.tmp/table/35978d5ff29248b595145b22b0c2554d as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/table/35978d5ff29248b595145b22b0c2554d 2024-11-13T05:37:49,007 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/table/35978d5ff29248b595145b22b0c2554d, entries=2, sequenceid=11, filesize=5.3 K 2024-11-13T05:37:49,008 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 154ms, sequenceid=11, compaction requested=false 2024-11-13T05:37:49,015 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T05:37:49,016 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:37:49,016 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:49,017 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476268854Running coprocessor pre-close hooks at 1731476268854Disabling compacts and flushes for region at 1731476268854Disabling writes for close at 1731476268854Obtaining lock to block concurrent updates at 1731476268855 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731476268855Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731476268855Flushing stores of hbase:meta,,1.1588230740 at 1731476268870 (+15 ms)Flushing 1588230740/info: creating writer at 1731476268870Flushing 1588230740/info: appending metadata at 1731476268887 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731476268887Flushing 1588230740/ns: creating writer at 1731476268911 (+24 ms)Flushing 1588230740/ns: appending metadata at 1731476268931 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731476268931Flushing 1588230740/table: creating writer at 1731476268949 (+18 ms)Flushing 1588230740/table: appending metadata at 1731476268965 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731476268965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70f628e0: reopening flushed file at 1731476268982 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3865ef96: reopening flushed file at 1731476268992 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52d72ff0: reopening flushed file at 1731476269000 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 154ms, sequenceid=11, compaction requested=false at 1731476269008 (+8 ms)Writing region close event to WAL at 1731476269012 (+4 ms)Running coprocessor post-close hooks at 1731476269016 (+4 ms)Closed at 1731476269016 2024-11-13T05:37:49,017 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:49,054 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,44475,1731476228646; all regions closed. 2024-11-13T05:37:49,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:49,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:49,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:49,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:49,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:49,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741894_1079 (size=825) 2024-11-13T05:37:49,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741894_1079 (size=825) 2024-11-13T05:37:49,388 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:37:49,389 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:37:49,486 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:49,486 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:37:49,486 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:37:50,165 INFO [master/a61c20bd729c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T05:37:50,165 INFO [master/a61c20bd729c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T05:37:50,322 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:50,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:50,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741836_1012 (size=76) 2024-11-13T05:37:51,392 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@4c1bae62[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45115, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45365, infoSecurePort=0, ipcPort=43351, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741828_1004 to 127.0.0.1:34231 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:51,392 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1d523e5c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45115, datanodeUuid=e9c7d410-ca24-42f8-8b74-27a3740d53c6, infoPort=45365, infoSecurePort=0, ipcPort=43351, storageInfo=lv=-57;cid=testClusterID;nsid=430027627;c=1731476226050):Failed to transfer BP-139784409-172.17.0.3-1731476226050:blk_1073741826_1002 to 127.0.0.1:34231 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741875_1058 (size=13268) 2024-11-13T05:37:52,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:52,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:52,862 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 after 4002ms 2024-11-13T05:37:52,871 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta after 4002ms 2024-11-13T05:37:53,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:53,860 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-13T05:37:53,862 DEBUG [RS:1;a61c20bd729c:35337 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs 2024-11-13T05:37:53,862 INFO [RS:1;a61c20bd729c:35337 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C35337%2C1731476230212:(num 1731476230452) 2024-11-13T05:37:53,863 DEBUG [RS:1;a61c20bd729c:35337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:37:53,863 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:53,863 INFO [RS:1;a61c20bd729c:35337 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35337 2024-11-13T05:37:53,869 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:53,912 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,35337,1731476230212 2024-11-13T05:37:53,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:53,914 INFO [RS:1;a61c20bd729c:35337 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:53,924 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,35337,1731476230212] 2024-11-13T05:37:53,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,931 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,932 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,935 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,35337,1731476230212 already deleted, retry=false 2024-11-13T05:37:53,935 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,35337,1731476230212 expired; onlineServers=1 2024-11-13T05:37:53,940 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:53,940 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:54,025 INFO [RS:1;a61c20bd729c:35337 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:54,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35337-0x10132b284fc0002, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:54,025 INFO [RS:1;a61c20bd729c:35337 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,35337,1731476230212; zookeeper connection closed. 2024-11-13T05:37:54,025 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e2425f0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e2425f0 2024-11-13T05:37:54,056 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-13T05:37:54,060 DEBUG [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs 2024-11-13T05:37:54,060 INFO [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C44475%2C1731476228646.meta:.meta(num 1731476268855) 2024-11-13T05:37:54,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741890_1074 (size=14682) 2024-11-13T05:37:54,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741890_1074 (size=14682) 2024-11-13T05:37:54,065 DEBUG [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs 2024-11-13T05:37:54,065 INFO [RS:0;a61c20bd729c:44475 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C44475%2C1731476228646:(num 1731476268515) 2024-11-13T05:37:54,065 DEBUG [RS:0;a61c20bd729c:44475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:37:54,065 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:37:54,065 INFO [RS:0;a61c20bd729c:44475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:54,066 INFO [RS:0;a61c20bd729c:44475 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:54,066 INFO [RS:0;a61c20bd729c:44475 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:54,066 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:54,066 INFO [RS:0;a61c20bd729c:44475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44475 2024-11-13T05:37:54,079 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,44475,1731476228646 2024-11-13T05:37:54,079 INFO [RS:0;a61c20bd729c:44475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:54,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:37:54,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,44475,1731476228646] 2024-11-13T05:37:54,100 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,44475,1731476228646 already deleted, retry=false 2024-11-13T05:37:54,100 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,44475,1731476228646 expired; onlineServers=0 2024-11-13T05:37:54,100 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,45157,1731476228451' ***** 2024-11-13T05:37:54,100 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:37:54,100 INFO [M:0;a61c20bd729c:45157 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:37:54,100 INFO [M:0;a61c20bd729c:45157 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:37:54,100 DEBUG [M:0;a61c20bd729c:45157 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:37:54,100 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:37:54,100 DEBUG [M:0;a61c20bd729c:45157 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:37:54,101 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476229342 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476229342,5,FailOnTimeoutGroup] 2024-11-13T05:37:54,101 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476229342 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476229342,5,FailOnTimeoutGroup] 2024-11-13T05:37:54,101 INFO [M:0;a61c20bd729c:45157 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:37:54,101 INFO [M:0;a61c20bd729c:45157 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:37:54,101 DEBUG [M:0;a61c20bd729c:45157 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:37:54,101 INFO [M:0;a61c20bd729c:45157 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:37:54,101 INFO [M:0;a61c20bd729c:45157 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:37:54,101 INFO [M:0;a61c20bd729c:45157 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:37:54,101 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:37:54,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:37:54,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:54,114 DEBUG [M:0;a61c20bd729c:45157 {}] zookeeper.ZKUtil(347): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:37:54,114 WARN [M:0;a61c20bd729c:45157 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:37:54,115 INFO [M:0;a61c20bd729c:45157 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/.lastflushedseqids 2024-11-13T05:37:54,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741902_1088 (size=130) 2024-11-13T05:37:54,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741902_1088 (size=130) 2024-11-13T05:37:54,121 INFO [M:0;a61c20bd729c:45157 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:37:54,122 INFO [M:0;a61c20bd729c:45157 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:37:54,122 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:37:54,122 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:54,122 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:54,122 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:37:54,122 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:54,122 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB 2024-11-13T05:37:54,144 DEBUG [M:0;a61c20bd729c:45157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d1ca6f136ca4ea297bed851ff352274 is 82, key is hbase:meta,,1/info:regioninfo/1731476230071/Put/seqid=0 2024-11-13T05:37:54,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741903_1089 (size=5672) 2024-11-13T05:37:54,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741903_1089 (size=5672) 2024-11-13T05:37:54,151 INFO [M:0;a61c20bd729c:45157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d1ca6f136ca4ea297bed851ff352274 2024-11-13T05:37:54,176 DEBUG [M:0;a61c20bd729c:45157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/79adc981b64d4d61814eb78eadd9ecb5 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731476230747/Put/seqid=0 2024-11-13T05:37:54,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741904_1090 (size=6255) 2024-11-13T05:37:54,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741904_1090 (size=6255) 2024-11-13T05:37:54,182 INFO [M:0;a61c20bd729c:45157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/79adc981b64d4d61814eb78eadd9ecb5 2024-11-13T05:37:54,189 INFO [M:0;a61c20bd729c:45157 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 79adc981b64d4d61814eb78eadd9ecb5 2024-11-13T05:37:54,190 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:54,190 INFO [RS:0;a61c20bd729c:44475 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:54,190 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44475-0x10132b284fc0001, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:54,190 INFO [RS:0;a61c20bd729c:44475 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,44475,1731476228646; zookeeper connection closed. 2024-11-13T05:37:54,190 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38f03070 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38f03070 2024-11-13T05:37:54,190 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-13T05:37:54,210 DEBUG [M:0;a61c20bd729c:45157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0da00b48a86a49c78e5f92cf9d5cdc11 is 69, key is a61c20bd729c,35337,1731476230212/rs:state/1731476230294/Put/seqid=0 2024-11-13T05:37:54,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741905_1091 (size=5224) 2024-11-13T05:37:54,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741905_1091 (size=5224) 2024-11-13T05:37:54,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:37:54,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,465 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,465 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,467 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:37:54,616 INFO [M:0;a61c20bd729c:45157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0da00b48a86a49c78e5f92cf9d5cdc11 2024-11-13T05:37:54,644 DEBUG [M:0;a61c20bd729c:45157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/afbb24c475144cc09e4cbba6999b6c4f is 52, key is load_balancer_on/state:d/1731476230195/Put/seqid=0 2024-11-13T05:37:54,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741906_1092 (size=5056) 2024-11-13T05:37:54,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741906_1092 (size=5056) 2024-11-13T05:37:54,649 INFO [M:0;a61c20bd729c:45157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/afbb24c475144cc09e4cbba6999b6c4f 2024-11-13T05:37:54,655 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d1ca6f136ca4ea297bed851ff352274 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d1ca6f136ca4ea297bed851ff352274 2024-11-13T05:37:54,662 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d1ca6f136ca4ea297bed851ff352274, entries=8, sequenceid=60, filesize=5.5 K 2024-11-13T05:37:54,663 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/79adc981b64d4d61814eb78eadd9ecb5 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/79adc981b64d4d61814eb78eadd9ecb5 2024-11-13T05:37:54,668 INFO [M:0;a61c20bd729c:45157 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 79adc981b64d4d61814eb78eadd9ecb5 2024-11-13T05:37:54,668 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/79adc981b64d4d61814eb78eadd9ecb5, entries=6, sequenceid=60, filesize=6.1 K 2024-11-13T05:37:54,670 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0da00b48a86a49c78e5f92cf9d5cdc11 as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0da00b48a86a49c78e5f92cf9d5cdc11 2024-11-13T05:37:54,675 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0da00b48a86a49c78e5f92cf9d5cdc11, entries=2, sequenceid=60, filesize=5.1 K 2024-11-13T05:37:54,676 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/afbb24c475144cc09e4cbba6999b6c4f as hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/afbb24c475144cc09e4cbba6999b6c4f 2024-11-13T05:37:54,682 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/afbb24c475144cc09e4cbba6999b6c4f, entries=1, sequenceid=60, filesize=4.9 K 2024-11-13T05:37:54,683 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 561ms, sequenceid=60, compaction requested=false 2024-11-13T05:37:54,685 INFO [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:54,685 DEBUG [M:0;a61c20bd729c:45157 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476274122Disabling compacts and flushes for region at 1731476274122Disabling writes for close at 1731476274122Obtaining lock to block concurrent updates at 1731476274122Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476274122Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731476274123 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476274123Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476274123Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476274144 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476274144Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476274157 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476274175 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476274175Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476274189 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476274209 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476274209Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476274623 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476274643 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476274643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49057c5e: reopening flushed file at 1731476274654 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b53a8b3: reopening flushed file at 1731476274662 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42e5cc22: reopening flushed file at 1731476274669 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5690abad: reopening flushed file at 1731476274675 (+6 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 561ms, sequenceid=60, compaction requested=false at 1731476274683 (+8 ms)Writing region close event to WAL at 1731476274685 (+2 ms)Closed at 1731476274685 2024-11-13T05:37:54,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:37:54,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45115 is added to blk_1073741888_1071 (size=1045) 2024-11-13T05:37:54,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40609 is added to blk_1073741888_1071 (size=1045) 2024-11-13T05:37:54,870 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:54,873 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:54,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-13T05:37:54,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:37:54,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:37:54,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:37:55,418 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3aa413f5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-139784409-172.17.0.3-1731476226050:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:36475,null,null]) java.net.ConnectException: Call From a61c20bd729c/172.17.0.3 to localhost:45147 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-13T05:37:55,871 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:55,874 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:56,363 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/WALs/a61c20bd729c,45157,1731476228451/a61c20bd729c%2C45157%2C1731476228451.1731476228799 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/oldWALs/a61c20bd729c%2C45157%2C1731476228451.1731476228799 2024-11-13T05:37:56,367 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/MasterData/oldWALs/a61c20bd729c%2C45157%2C1731476228451.1731476228799 to hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/oldWALs/a61c20bd729c%2C45157%2C1731476228451.1731476228799$masterlocalwal$ 2024-11-13T05:37:56,367 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:37:56,368 INFO [M:0;a61c20bd729c:45157 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:37:56,368 INFO [M:0;a61c20bd729c:45157 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45157 2024-11-13T05:37:56,368 INFO [M:0;a61c20bd729c:45157 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:37:56,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:56,657 INFO [M:0;a61c20bd729c:45157 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:37:56,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45157-0x10132b284fc0000, quorum=127.0.0.1:51491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:37:56,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@500e0366{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:56,662 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7900579e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:56,662 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:56,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58dca5c7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:56,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76c419ee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:56,663 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:56,663 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:56,663 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-139784409-172.17.0.3-1731476226050 (Datanode Uuid e9c7d410-ca24-42f8-8b74-27a3740d53c6) service to localhost/127.0.0.1:41149 2024-11-13T05:37:56,663 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:56,663 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@42282233 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:36475,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:45147 , LocalHost:localPort a61c20bd729c/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-13T05:37:56,664 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@42282233 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:45115,null,null]) java.io.IOException: No block pool offer service for bpid=BP-139784409-172.17.0.3-1731476226050 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:56,664 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@42282233 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:36475,null,null], DatanodeInfoWithStorage[127.0.0.1:45115,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-139784409-172.17.0.3-1731476226050:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:36475,null,null], DatanodeInfoWithStorage[127.0.0.1:45115,null,null]] 2024-11-13T05:37:56,664 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@42282233 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:36475,null,null]) java.io.IOException: No block pool offer service for bpid=BP-139784409-172.17.0.3-1731476226050 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:56,664 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data3/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:56,664 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@42282233 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:45115,null,null]) java.io.IOException: No block pool offer service for bpid=BP-139784409-172.17.0.3-1731476226050 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:37:56,664 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@42282233 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:36475,null,null], DatanodeInfoWithStorage[127.0.0.1:45115,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-139784409-172.17.0.3-1731476226050:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:36475,null,null], DatanodeInfoWithStorage[127.0.0.1:45115,null,null]] 2024-11-13T05:37:56,665 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data4/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:56,665 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:56,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@580ed5d9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:56,672 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7645500{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:56,672 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:56,672 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d01e731{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:56,672 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a2ad893{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:56,679 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:37:56,679 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:37:56,679 WARN [BP-139784409-172.17.0.3-1731476226050 heartbeating to localhost/127.0.0.1:41149 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-139784409-172.17.0.3-1731476226050 (Datanode Uuid efbb409e-f0e5-4411-a663-f71f252b3629) service to localhost/127.0.0.1:41149 2024-11-13T05:37:56,679 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:37:56,679 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data5/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:56,680 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/cluster_a5689c72-caca-4fbe-ee26-5feb0b179604/data/data6/current/BP-139784409-172.17.0.3-1731476226050 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:37:56,680 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:37:56,690 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@acc902f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:37:56,692 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71ccb409{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:37:56,692 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:37:56,693 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c3e366f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:37:56,693 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@100caf4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir/,STOPPED} 2024-11-13T05:37:56,702 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:37:56,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:37:56,744 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 78) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:41149 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41149 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:36205 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41149 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:41149 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:36205 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41149 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41149 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41149 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41149 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6b70bf4210.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41149 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6b70bf4210.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41149 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41149 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 188) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3327 (was 2814) - AvailableMemoryMB LEAK? - 2024-11-13T05:37:56,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=3327 2024-11-13T05:37:56,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:37:56,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.log.dir so I do NOT create it in target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50 2024-11-13T05:37:56,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/41fce870-7d0d-2490-6fc1-b23614e115f0/hadoop.tmp.dir so I do NOT create it in target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50 2024-11-13T05:37:56,752 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2, deleteOnExit=true 2024-11-13T05:37:56,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/test.cache.data in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:37:56,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:37:56,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:37:56,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:37:56,770 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:37:56,872 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:56,875 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:57,260 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:57,264 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:57,270 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:57,270 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:57,270 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:37:57,271 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:57,271 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69a0d4c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:57,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30acbc29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:57,367 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@424e904a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-42929-hadoop-hdfs-3_4_1-tests_jar-_-any-11894640763862818365/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:37:57,368 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6d7adad9{HTTP/1.1, (http/1.1)}{localhost:42929} 2024-11-13T05:37:57,368 INFO [Time-limited test {}] server.Server(415): Started @159336ms 2024-11-13T05:37:57,381 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:37:57,873 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:57,875 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:57,959 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:57,962 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:57,963 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:57,963 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:57,963 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:57,964 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e82216{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:57,964 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7661aa7f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:58,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a71e43c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-44633-hadoop-hdfs-3_4_1-tests_jar-_-any-11982701186826490616/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:58,065 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@267d44f1{HTTP/1.1, (http/1.1)}{localhost:44633} 2024-11-13T05:37:58,065 INFO [Time-limited test {}] server.Server(415): Started @160033ms 2024-11-13T05:37:58,067 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:58,097 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:37:58,100 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:37:58,101 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:37:58,101 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:37:58,101 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:37:58,102 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8d1e34{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:37:58,102 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@60067fa0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:37:58,205 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@15b808c5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-37197-hadoop-hdfs-3_4_1-tests_jar-_-any-11305664459301294704/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:37:58,211 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@129a0927{HTTP/1.1, (http/1.1)}{localhost:37197} 2024-11-13T05:37:58,211 INFO [Time-limited test {}] server.Server(415): Started @160179ms 2024-11-13T05:37:58,213 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:37:58,874 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:58,876 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:59,188 WARN [Thread-1203 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data2/current/BP-1384789590-172.17.0.3-1731476276781/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:59,188 WARN [Thread-1202 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data1/current/BP-1384789590-172.17.0.3-1731476276781/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:59,210 WARN [Thread-1166 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x56a19b2eb7ee531e with lease ID 0xc68a041922a26ff6: Processing first storage report for DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f from datanode DatanodeRegistration(127.0.0.1:44119, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=43201, infoSecurePort=0, ipcPort=35863, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781) 2024-11-13T05:37:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56a19b2eb7ee531e with lease ID 0xc68a041922a26ff6: from storage DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f node DatanodeRegistration(127.0.0.1:44119, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=43201, infoSecurePort=0, ipcPort=35863, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:37:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x56a19b2eb7ee531e with lease ID 0xc68a041922a26ff6: Processing first storage report for DS-f37e78f9-6a14-42cf-8d54-3e03dec91bd5 from datanode DatanodeRegistration(127.0.0.1:44119, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=43201, infoSecurePort=0, ipcPort=35863, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781) 2024-11-13T05:37:59,213 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56a19b2eb7ee531e with lease ID 0xc68a041922a26ff6: from storage DS-f37e78f9-6a14-42cf-8d54-3e03dec91bd5 node DatanodeRegistration(127.0.0.1:44119, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=43201, infoSecurePort=0, ipcPort=35863, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:59,327 WARN [Thread-1213 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data3/current/BP-1384789590-172.17.0.3-1731476276781/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:59,327 WARN [Thread-1214 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data4/current/BP-1384789590-172.17.0.3-1731476276781/current, will proceed with Du for space computation calculation, 2024-11-13T05:37:59,346 WARN [Thread-1189 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:37:59,348 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb7abb506805d32f7 with lease ID 0xc68a041922a26ff7: Processing first storage report for DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b from datanode DatanodeRegistration(127.0.0.1:35135, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=42089, infoSecurePort=0, ipcPort=33319, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781) 2024-11-13T05:37:59,349 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb7abb506805d32f7 with lease ID 0xc68a041922a26ff7: from storage DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b node DatanodeRegistration(127.0.0.1:35135, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=42089, infoSecurePort=0, ipcPort=33319, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:59,349 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb7abb506805d32f7 with lease ID 0xc68a041922a26ff7: Processing first storage report for DS-047d6be3-fee1-4df2-817d-2b08cb5b63ff from datanode DatanodeRegistration(127.0.0.1:35135, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=42089, infoSecurePort=0, ipcPort=33319, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781) 2024-11-13T05:37:59,349 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb7abb506805d32f7 with lease ID 0xc68a041922a26ff7: from storage DS-047d6be3-fee1-4df2-817d-2b08cb5b63ff node DatanodeRegistration(127.0.0.1:35135, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=42089, infoSecurePort=0, ipcPort=33319, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:37:59,352 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50 2024-11-13T05:37:59,355 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/zookeeper_0, clientPort=57910, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:37:59,356 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57910 2024-11-13T05:37:59,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,357 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:37:59,366 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39 with version=8 2024-11-13T05:37:59,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:37:59,368 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:59,368 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:59,369 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41685 2024-11-13T05:37:59,371 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41685 connecting to ZooKeeper ensemble=127.0.0.1:57910 2024-11-13T05:37:59,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416850x0, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:59,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41685-0x10132b34be50000 connected 2024-11-13T05:37:59,512 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:59,520 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39, hbase.cluster.distributed=false 2024-11-13T05:37:59,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:59,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41685 2024-11-13T05:37:59,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41685 2024-11-13T05:37:59,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41685 2024-11-13T05:37:59,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41685 2024-11-13T05:37:59,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41685 2024-11-13T05:37:59,539 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:37:59,540 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:37:59,541 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39593 2024-11-13T05:37:59,542 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39593 connecting to ZooKeeper ensemble=127.0.0.1:57910 2024-11-13T05:37:59,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,544 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395930x0, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:37:59,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:37:59,554 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39593-0x10132b34be50001 connected 2024-11-13T05:37:59,554 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:37:59,555 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:37:59,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:37:59,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:37:59,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39593 2024-11-13T05:37:59,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39593 2024-11-13T05:37:59,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39593 2024-11-13T05:37:59,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39593 2024-11-13T05:37:59,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39593 2024-11-13T05:37:59,571 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:41685 2024-11-13T05:37:59,571 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:59,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:59,575 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:37:59,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,585 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:37:59,586 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,41685,1731476279368 from backup master directory 2024-11-13T05:37:59,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:59,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:37:59,595 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:37:59,595 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,602 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/hbase.id] with ID: 8799e589-3634-458f-b9c5-5f15542479bb 2024-11-13T05:37:59,602 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/.tmp/hbase.id 2024-11-13T05:37:59,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:37:59,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:37:59,610 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/.tmp/hbase.id]:[hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/hbase.id] 2024-11-13T05:37:59,628 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:37:59,628 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:37:59,631 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-13T05:37:59,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:37:59,649 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:37:59,650 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:37:59,650 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:37:59,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:37:59,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:37:59,657 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store 2024-11-13T05:37:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:37:59,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:59,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:37:59,666 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:59,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:59,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:37:59,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:59,666 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:37:59,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476279666Disabling compacts and flushes for region at 1731476279666Disabling writes for close at 1731476279666Writing region close event to WAL at 1731476279666Closed at 1731476279666 2024-11-13T05:37:59,667 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/.initializing 2024-11-13T05:37:59,667 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,670 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C41685%2C1731476279368, suffix=, logDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368, archiveDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/oldWALs, maxLogs=10 2024-11-13T05:37:59,670 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C41685%2C1731476279368.1731476279670 2024-11-13T05:37:59,675 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 2024-11-13T05:37:59,675 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42089:42089),(127.0.0.1/127.0.0.1:43201:43201)] 2024-11-13T05:37:59,676 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:37:59,676 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:59,676 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,676 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:37:59,682 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:59,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:37:59,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:59,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:37:59,685 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:59,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,686 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:37:59,687 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:37:59,687 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,688 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,688 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,689 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,689 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,690 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:37:59,691 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:37:59,693 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:59,694 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713400, jitterRate=-0.09286503493785858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:37:59,694 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476279676Initializing all the Stores at 1731476279677 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476279677Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476279680 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476279680Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476279680Cleaning up temporary data from old regions at 1731476279689 (+9 ms)Region opened successfully at 1731476279694 (+5 ms) 2024-11-13T05:37:59,694 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:37:59,697 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7816a766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:59,698 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:37:59,698 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:37:59,698 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:37:59,698 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:37:59,699 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:37:59,699 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:37:59,699 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:37:59,702 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:37:59,702 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:37:59,711 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:37:59,711 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:37:59,712 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:37:59,722 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:37:59,722 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:37:59,723 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:37:59,732 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:37:59,734 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:37:59,743 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:37:59,745 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:37:59,753 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:37:59,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:59,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:37:59,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,764 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,41685,1731476279368, sessionid=0x10132b34be50000, setting cluster-up flag (Was=false) 2024-11-13T05:37:59,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,817 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:37:59,818 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:37:59,869 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:37:59,870 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,41685,1731476279368 2024-11-13T05:37:59,871 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:37:59,873 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:59,873 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:37:59,873 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:37:59,873 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,41685,1731476279368 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:37:59,874 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:37:59,875 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:37:59,876 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476309876 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:37:59,877 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:59,878 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:59,878 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:37:59,878 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476279878,5,FailOnTimeoutGroup] 2024-11-13T05:37:59,878 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476279878,5,FailOnTimeoutGroup] 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:59,878 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:37:59,879 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:59,879 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:37:59,879 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,879 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:37:59,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:37:59,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:37:59,889 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:37:59,889 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39 2024-11-13T05:37:59,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:59,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:37:59,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:37:59,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:37:59,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:37:59,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:59,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:37:59,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:37:59,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:59,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:37:59,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:37:59,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:59,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:37:59,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:37:59,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:37:59,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:37:59,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:37:59,909 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740 2024-11-13T05:37:59,909 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740 2024-11-13T05:37:59,910 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:37:59,910 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:37:59,911 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:37:59,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:37:59,914 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:37:59,914 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=696579, jitterRate=-0.11425511538982391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:37:59,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476279900Initializing all the Stores at 1731476279901 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476279901Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476279901Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476279901Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476279901Cleaning up temporary data from old regions at 1731476279910 (+9 ms)Region opened successfully at 1731476279914 (+4 ms) 2024-11-13T05:37:59,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:37:59,915 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:37:59,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:37:59,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:37:59,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:37:59,915 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:37:59,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476279915Disabling compacts and flushes for region at 1731476279915Disabling writes for close at 1731476279915Writing region close event to WAL at 1731476279915Closed at 1731476279915 2024-11-13T05:37:59,916 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:37:59,917 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:37:59,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:37:59,918 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:37:59,919 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:37:59,960 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(746): ClusterId : 8799e589-3634-458f-b9c5-5f15542479bb 2024-11-13T05:37:59,960 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:37:59,965 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:37:59,965 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:37:59,975 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:37:59,976 DEBUG [RS:0;a61c20bd729c:39593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549a90d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:37:59,988 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:39593 2024-11-13T05:37:59,988 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:37:59,988 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:37:59,988 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:37:59,989 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,41685,1731476279368 with port=39593, startcode=1731476279539 2024-11-13T05:37:59,989 DEBUG [RS:0;a61c20bd729c:39593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:37:59,991 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49337, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:37:59,992 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41685 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,39593,1731476279539 2024-11-13T05:37:59,992 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41685 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,39593,1731476279539 2024-11-13T05:37:59,994 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39 2024-11-13T05:37:59,994 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43693 2024-11-13T05:37:59,994 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:38:00,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:38:00,006 DEBUG [RS:0;a61c20bd729c:39593 {}] zookeeper.ZKUtil(111): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,006 WARN [RS:0;a61c20bd729c:39593 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:38:00,006 INFO [RS:0;a61c20bd729c:39593 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:38:00,007 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,007 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,39593,1731476279539] 2024-11-13T05:38:00,010 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:38:00,011 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:38:00,012 INFO [RS:0;a61c20bd729c:39593 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:38:00,012 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,012 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:38:00,013 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:38:00,013 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:38:00,013 DEBUG [RS:0;a61c20bd729c:39593 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:38:00,015 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,015 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,015 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,015 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,015 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,016 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,39593,1731476279539-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:38:00,030 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:38:00,030 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,39593,1731476279539-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,030 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,030 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.Replication(171): a61c20bd729c,39593,1731476279539 started 2024-11-13T05:38:00,043 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,043 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,39593,1731476279539, RpcServer on a61c20bd729c/172.17.0.3:39593, sessionid=0x10132b34be50001 2024-11-13T05:38:00,043 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:38:00,043 DEBUG [RS:0;a61c20bd729c:39593 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,043 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,39593,1731476279539' 2024-11-13T05:38:00,043 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:38:00,044 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:38:00,044 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:38:00,044 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:38:00,044 DEBUG [RS:0;a61c20bd729c:39593 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,044 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,39593,1731476279539' 2024-11-13T05:38:00,044 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:38:00,045 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:38:00,045 DEBUG [RS:0;a61c20bd729c:39593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:38:00,045 INFO [RS:0;a61c20bd729c:39593 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:38:00,045 INFO [RS:0;a61c20bd729c:39593 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:38:00,070 WARN [a61c20bd729c:41685 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:38:00,148 INFO [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C39593%2C1731476279539, suffix=, logDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539, archiveDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs, maxLogs=32 2024-11-13T05:38:00,148 INFO [RS:0;a61c20bd729c:39593 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:00,154 INFO [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:00,159 DEBUG [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43201:43201),(127.0.0.1/127.0.0.1:42089:42089)] 2024-11-13T05:38:00,320 DEBUG [a61c20bd729c:41685 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:38:00,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,322 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,39593,1731476279539, state=OPENING 2024-11-13T05:38:00,395 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:38:00,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:00,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:00,406 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:38:00,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:00,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:00,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,39593,1731476279539}] 2024-11-13T05:38:00,472 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:38:00,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,494 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,494 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,495 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,499 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:00,559 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:38:00,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49031, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:38:00,566 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:38:00,566 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:38:00,568 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C39593%2C1731476279539.meta, suffix=.meta, logDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539, archiveDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs, maxLogs=32 2024-11-13T05:38:00,569 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta 2024-11-13T05:38:00,577 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta 2024-11-13T05:38:00,578 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42089:42089),(127.0.0.1/127.0.0.1:43201:43201)] 2024-11-13T05:38:00,584 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:38:00,584 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:38:00,584 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:38:00,584 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:38:00,585 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:38:00,585 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:00,585 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:38:00,585 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:38:00,586 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:38:00,587 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:38:00,587 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:00,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:00,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:38:00,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:38:00,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:00,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:00,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:38:00,591 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:38:00,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:00,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:00,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:38:00,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:38:00,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:00,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:00,593 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:38:00,594 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740 2024-11-13T05:38:00,595 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740 2024-11-13T05:38:00,596 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:38:00,596 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:38:00,597 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:38:00,598 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:38:00,599 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771982, jitterRate=-0.01837506890296936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:38:00,599 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:38:00,600 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476280585Writing region info on filesystem at 1731476280585Initializing all the Stores at 1731476280586 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476280586Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476280586Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476280586Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476280586Cleaning up temporary data from old regions at 1731476280596 (+10 ms)Running coprocessor post-open hooks at 1731476280600 (+4 ms)Region opened successfully at 1731476280600 2024-11-13T05:38:00,601 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476280559 2024-11-13T05:38:00,604 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:38:00,604 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:38:00,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,605 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,39593,1731476279539, state=OPEN 2024-11-13T05:38:00,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:38:00,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:38:00,647 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:00,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:00,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:38:00,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,39593,1731476279539 in 241 msec 2024-11-13T05:38:00,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:38:00,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 733 msec 2024-11-13T05:38:00,654 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:38:00,654 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:38:00,656 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:38:00,656 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,39593,1731476279539, seqNum=-1] 2024-11-13T05:38:00,657 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:38:00,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50565, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:38:00,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 792 msec 2024-11-13T05:38:00,665 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476280665, completionTime=-1 2024-11-13T05:38:00,665 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:38:00,666 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:38:00,668 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:38:00,668 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476340668 2024-11-13T05:38:00,668 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476400668 2024-11-13T05:38:00,668 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec 2024-11-13T05:38:00,669 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41685,1731476279368-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,669 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41685,1731476279368-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,669 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41685,1731476279368-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,669 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:41685, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,669 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,669 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,671 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:38:00,673 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.078sec 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41685,1731476279368-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:38:00,674 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41685,1731476279368-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:38:00,676 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:38:00,676 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:38:00,676 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,41685,1731476279368-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:00,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f69ea93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:38:00,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,41685,-1 for getting cluster id 2024-11-13T05:38:00,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:38:00,762 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8799e589-3634-458f-b9c5-5f15542479bb' 2024-11-13T05:38:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:38:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8799e589-3634-458f-b9c5-5f15542479bb" 2024-11-13T05:38:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7ff723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:38:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,41685,-1] 2024-11-13T05:38:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:38:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:00,765 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:38:00,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689abc03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:38:00,766 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:38:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,39593,1731476279539, seqNum=-1] 2024-11-13T05:38:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:38:00,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:38:00,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,41685,1731476279368 2024-11-13T05:38:00,772 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:00,775 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:38:00,775 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-11-13T05:38:00,775 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-11-13T05:38:00,775 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:38:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is a61c20bd729c,41685,1731476279368 2024-11-13T05:38:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@177938c4 2024-11-13T05:38:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:38:00,779 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:38:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-13T05:38:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-13T05:38:00,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:38:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-11-13T05:38:00,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:38:00,783 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:00,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-11-13T05:38:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:38:00,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:38:00,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741835_1011 (size=395) 2024-11-13T05:38:00,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741835_1011 (size=395) 2024-11-13T05:38:00,803 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6870496ed7dea82a64bc12c5235753c1, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39 2024-11-13T05:38:00,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35135 is added to blk_1073741836_1012 (size=78) 2024-11-13T05:38:00,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44119 is added to blk_1073741836_1012 (size=78) 2024-11-13T05:38:00,815 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:00,815 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 6870496ed7dea82a64bc12c5235753c1, disabling compactions & flushes 2024-11-13T05:38:00,815 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:00,815 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:00,815 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. after waiting 0 ms 2024-11-13T05:38:00,815 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:00,815 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:00,815 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6870496ed7dea82a64bc12c5235753c1: Waiting for close lock at 1731476280815Disabling compacts and flushes for region at 1731476280815Disabling writes for close at 1731476280815Writing region close event to WAL at 1731476280815Closed at 1731476280815 2024-11-13T05:38:00,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:38:00,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731476280818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476280818"}]},"ts":"1731476280818"} 2024-11-13T05:38:00,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:38:00,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:38:00,822 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476280822"}]},"ts":"1731476280822"} 2024-11-13T05:38:00,825 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-11-13T05:38:00,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=6870496ed7dea82a64bc12c5235753c1, ASSIGN}] 2024-11-13T05:38:00,827 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=6870496ed7dea82a64bc12c5235753c1, ASSIGN 2024-11-13T05:38:00,828 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=6870496ed7dea82a64bc12c5235753c1, ASSIGN; state=OFFLINE, location=a61c20bd729c,39593,1731476279539; forceNewPlan=false, retain=false 2024-11-13T05:38:00,875 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:00,877 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:00,979 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6870496ed7dea82a64bc12c5235753c1, regionState=OPENING, regionLocation=a61c20bd729c,39593,1731476279539 2024-11-13T05:38:00,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=6870496ed7dea82a64bc12c5235753c1, ASSIGN because future has completed 2024-11-13T05:38:00,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6870496ed7dea82a64bc12c5235753c1, server=a61c20bd729c,39593,1731476279539}] 2024-11-13T05:38:01,139 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:01,139 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6870496ed7dea82a64bc12c5235753c1, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:38:01,140 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,140 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:01,140 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,140 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,141 INFO [StoreOpener-6870496ed7dea82a64bc12c5235753c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,143 INFO [StoreOpener-6870496ed7dea82a64bc12c5235753c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6870496ed7dea82a64bc12c5235753c1 columnFamilyName info 2024-11-13T05:38:01,143 DEBUG [StoreOpener-6870496ed7dea82a64bc12c5235753c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:01,143 INFO [StoreOpener-6870496ed7dea82a64bc12c5235753c1-1 {}] regionserver.HStore(327): Store=6870496ed7dea82a64bc12c5235753c1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:38:01,143 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,144 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,144 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,145 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,145 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,147 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,149 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:38:01,150 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6870496ed7dea82a64bc12c5235753c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=718718, jitterRate=-0.08610381186008453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:38:01,150 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:01,150 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6870496ed7dea82a64bc12c5235753c1: Running coprocessor pre-open hook at 1731476281140Writing region info on filesystem at 1731476281140Initializing all the Stores at 1731476281141 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476281141Cleaning up temporary data from old regions at 1731476281145 (+4 ms)Running coprocessor post-open hooks at 1731476281150 (+5 ms)Region opened successfully at 1731476281150 2024-11-13T05:38:01,152 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1., pid=6, masterSystemTime=1731476281135 2024-11-13T05:38:01,155 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:01,155 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:01,156 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6870496ed7dea82a64bc12c5235753c1, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,39593,1731476279539 2024-11-13T05:38:01,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6870496ed7dea82a64bc12c5235753c1, server=a61c20bd729c,39593,1731476279539 because future has completed 2024-11-13T05:38:01,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:38:01,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6870496ed7dea82a64bc12c5235753c1, server=a61c20bd729c,39593,1731476279539 in 178 msec 2024-11-13T05:38:01,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:38:01,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=6870496ed7dea82a64bc12c5235753c1, ASSIGN in 339 msec 2024-11-13T05:38:01,168 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:38:01,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476281168"}]},"ts":"1731476281168"} 2024-11-13T05:38:01,171 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-11-13T05:38:01,172 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:38:01,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 392 msec 2024-11-13T05:38:01,876 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:01,877 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:02,877 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:02,878 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:03,878 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:03,879 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:04,879 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:04,880 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:04,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:38:04,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:38:04,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-13T05:38:04,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-11-13T05:38:04,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:38:04,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:38:04,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:38:04,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:38:05,880 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:05,880 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:06,091 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:38:06,112 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,113 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,113 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,113 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,114 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,114 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,118 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,118 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,118 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:06,125 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:38:06,126 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-11-13T05:38:06,881 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:06,882 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:07,882 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:07,882 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:08,883 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:08,883 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:09,884 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:09,884 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:38:10,796 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-11-13T05:38:10,796 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-11-13T05:38:10,802 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-11-13T05:38:10,802 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1., hostname=a61c20bd729c,39593,1731476279539, seqNum=2] 2024-11-13T05:38:10,885 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:10,885 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:11,887 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:11,887 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:12,812 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:12,813 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:12,813 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:12,813 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:12,814 WARN [DataStreamer for file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 block BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK], DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]) is bad. 2024-11-13T05:38:12,814 WARN [DataStreamer for file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta block BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK], DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]) is bad. 2024-11-13T05:38:12,814 WARN [DataStreamer for file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 block BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK], DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35135,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]) is bad. 2024-11-13T05:38:12,815 WARN [PacketResponder: BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35135] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,815 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:34140 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35135:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34140 dst: /127.0.0.1:35135 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,815 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1745605868_22 at /127.0.0.1:34102 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35135:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34102 dst: /127.0.0.1:35135 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:44450 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44450 dst: /127.0.0.1:44119 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:44436 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44436 dst: /127.0.0.1:44119 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1745605868_22 at /127.0.0.1:44418 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44418 dst: /127.0.0.1:44119 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:34124 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35135:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34124 dst: /127.0.0.1:35135 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:12,888 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:12,888 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:12,897 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@15b808c5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:12,897 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@129a0927{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:12,897 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:12,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@60067fa0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:12,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8d1e34{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:12,899 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:38:12,899 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:38:12,899 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1384789590-172.17.0.3-1731476276781 (Datanode Uuid 500baa8a-43de-463b-974c-f32c261fb587) service to localhost/127.0.0.1:43693 2024-11-13T05:38:12,899 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:38:12,899 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data3/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:12,899 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data4/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:12,900 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:38:12,906 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:12,909 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:12,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:12,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:12,910 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:38:12,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b503321{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:12,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74025bad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:13,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@33b84383{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-41509-hadoop-hdfs-3_4_1-tests_jar-_-any-16650739529846323415/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:13,007 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c02cba1{HTTP/1.1, (http/1.1)}{localhost:41509} 2024-11-13T05:38:13,007 INFO [Time-limited test {}] server.Server(415): Started @174975ms 2024-11-13T05:38:13,008 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:38:13,023 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:13,023 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:13,023 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:13,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1745605868_22 at /127.0.0.1:41220 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41220 dst: /127.0.0.1:44119 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:13,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:41216 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41216 dst: /127.0.0.1:44119 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:13,024 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:41214 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41214 dst: /127.0.0.1:44119 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:13,025 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a71e43c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:13,026 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@267d44f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:13,026 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:13,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7661aa7f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:13,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e82216{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:13,027 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:38:13,027 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:38:13,027 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1384789590-172.17.0.3-1731476276781 (Datanode Uuid ac95cf31-264e-4108-9356-b85ca1914e9f) service to localhost/127.0.0.1:43693 2024-11-13T05:38:13,027 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:38:13,027 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data1/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:13,028 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data2/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:13,028 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:38:13,039 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:13,043 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:13,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:13,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:13,044 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:38:13,044 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b86e250{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:13,045 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62498b6c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:13,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48f4fa87{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-36951-hadoop-hdfs-3_4_1-tests_jar-_-any-4222259264972145263/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:13,140 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3581a063{HTTP/1.1, (http/1.1)}{localhost:36951} 2024-11-13T05:38:13,140 INFO [Time-limited test {}] server.Server(415): Started @175108ms 2024-11-13T05:38:13,142 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:38:13,585 WARN [Thread-1338 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:38:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80f7141de03a9f0c with lease ID 0xc68a041922a26ff8: from storage DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b node DatanodeRegistration(127.0.0.1:44891, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=44549, infoSecurePort=0, ipcPort=37179, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80f7141de03a9f0c with lease ID 0xc68a041922a26ff8: from storage DS-047d6be3-fee1-4df2-817d-2b08cb5b63ff node DatanodeRegistration(127.0.0.1:44891, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=44549, infoSecurePort=0, ipcPort=37179, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:13,796 WARN [Thread-1358 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:38:13,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3584bfcfb493334 with lease ID 0xc68a041922a26ff9: from storage DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f node DatanodeRegistration(127.0.0.1:35215, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=37657, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:13,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3584bfcfb493334 with lease ID 0xc68a041922a26ff9: from storage DS-f37e78f9-6a14-42cf-8d54-3e03dec91bd5 node DatanodeRegistration(127.0.0.1:35215, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=37657, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:13,889 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:13,889 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:14,166 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-11-13T05:38:14,172 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-11-13T05:38:14,175 ERROR [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:14,175 WARN [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:14,175 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C39593%2C1731476279539:(num 1731476280148) roll requested 2024-11-13T05:38:14,176 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:14,182 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 newFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:14,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:14,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:14,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:14,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:14,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:14,183 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:14,183 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:14,183 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:14,183 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:14,183 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37657:37657),(127.0.0.1/127.0.0.1:44549:44549)] 2024-11-13T05:38:14,183 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 is not closed yet, will try archiving it next time 2024-11-13T05:38:14,184 WARN [IPC Server handler 2 on default port 43693 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014 2024-11-13T05:38:14,184 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 after 1ms 2024-11-13T05:38:14,890 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:14,890 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:15,891 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:15,891 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:16,190 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-11-13T05:38:16,588 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-13T05:38:16,893 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:16,893 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:17,894 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:17,894 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:18,185 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 after 4002ms 2024-11-13T05:38:18,194 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:18,195 WARN [DataStreamer for file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 block BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35215,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK], DatanodeInfoWithStorage[127.0.0.1:44891,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35215,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]) is bad. 2024-11-13T05:38:18,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:48778 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:35215:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48778 dst: /127.0.0.1:35215 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:18,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:40796 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:44891:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40796 dst: /127.0.0.1:44891 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:18,276 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48f4fa87{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:18,276 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3581a063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:18,276 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:18,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62498b6c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:18,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b86e250{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:18,278 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:38:18,278 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:38:18,278 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:38:18,278 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1384789590-172.17.0.3-1731476276781 (Datanode Uuid ac95cf31-264e-4108-9356-b85ca1914e9f) service to localhost/127.0.0.1:43693 2024-11-13T05:38:18,278 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data1/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:18,279 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data2/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:18,279 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:38:18,288 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:18,291 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:18,292 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:18,292 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:18,292 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:38:18,292 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47ddff64{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:18,293 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6edb8d2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:18,390 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d04913a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-45369-hadoop-hdfs-3_4_1-tests_jar-_-any-13261132626818532647/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:18,390 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@263c339e{HTTP/1.1, (http/1.1)}{localhost:45369} 2024-11-13T05:38:18,390 INFO [Time-limited test {}] server.Server(415): Started @180358ms 2024-11-13T05:38:18,391 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:38:18,410 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:18,411 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-459126326_22 at /127.0.0.1:40806 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:44891:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40806 dst: /127.0.0.1:44891 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:18,417 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@33b84383{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:18,417 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c02cba1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:18,417 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:18,417 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74025bad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:18,418 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b503321{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:18,419 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:38:18,419 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:38:18,419 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:38:18,419 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1384789590-172.17.0.3-1731476276781 (Datanode Uuid 500baa8a-43de-463b-974c-f32c261fb587) service to localhost/127.0.0.1:43693 2024-11-13T05:38:18,419 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data3/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:18,420 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data4/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:18,420 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:38:18,434 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:18,438 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:18,438 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:18,438 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:18,439 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:38:18,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2cd6e960{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:18,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@324d7520{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:18,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55b58292{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/java.io.tmpdir/jetty-localhost-39821-hadoop-hdfs-3_4_1-tests_jar-_-any-3479530904286079749/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:18,533 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@90a8632{HTTP/1.1, (http/1.1)}{localhost:39821} 2024-11-13T05:38:18,533 INFO [Time-limited test {}] server.Server(415): Started @180501ms 2024-11-13T05:38:18,535 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:38:18,895 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:18,895 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:18,933 WARN [Thread-1412 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:38:18,935 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x692a31a5b05cd45f with lease ID 0xc68a041922a26ffa: from storage DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f node DatanodeRegistration(127.0.0.1:33569, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=45019, infoSecurePort=0, ipcPort=37667, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:38:18,935 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x692a31a5b05cd45f with lease ID 0xc68a041922a26ffa: from storage DS-f37e78f9-6a14-42cf-8d54-3e03dec91bd5 node DatanodeRegistration(127.0.0.1:33569, datanodeUuid=ac95cf31-264e-4108-9356-b85ca1914e9f, infoPort=45019, infoSecurePort=0, ipcPort=37667, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:19,100 WARN [Thread-1432 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:38:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x522d9ff0e14c5ea3 with lease ID 0xc68a041922a26ffb: from storage DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b node DatanodeRegistration(127.0.0.1:35013, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=38971, infoSecurePort=0, ipcPort=46821, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:38:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x522d9ff0e14c5ea3 with lease ID 0xc68a041922a26ffb: from storage DS-047d6be3-fee1-4df2-817d-2b08cb5b63ff node DatanodeRegistration(127.0.0.1:35013, datanodeUuid=500baa8a-43de-463b-974c-f32c261fb587, infoPort=38971, infoSecurePort=0, ipcPort=46821, storageInfo=lv=-57;cid=testClusterID;nsid=539354575;c=1731476276781), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:19,553 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-11-13T05:38:19,558 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-11-13T05:38:19,560 ERROR [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44891,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:19,561 WARN [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44891,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:19,561 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C39593%2C1731476279539:(num 1731476294175) roll requested 2024-11-13T05:38:19,561 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.1731476299561 2024-11-13T05:38:19,568 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 newFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 2024-11-13T05:38:19,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:19,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:19,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:19,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:19,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:19,569 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 2024-11-13T05:38:19,569 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44891,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:19,569 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44891,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:19,569 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:19,570 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:38971:38971)] 2024-11-13T05:38:19,570 WARN [IPC Server handler 0 on default port 43693 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-11-13T05:38:19,570 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 is not closed yet, will try archiving it next time 2024-11-13T05:38:19,570 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 after 1ms 2024-11-13T05:38:19,896 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:19,896 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:20,897 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:20,897 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:21,571 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:21,576 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 newFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:21,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:21,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:21,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:21,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:21,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:21,577 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:21,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:38971:38971)] 2024-11-13T05:38:21,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 is not closed yet, will try archiving it next time 2024-11-13T05:38:21,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 is not closed yet, will try archiving it next time 2024-11-13T05:38:21,578 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:21,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:21,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 after 1ms 2024-11-13T05:38:21,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741838_1019 (size=1264) 2024-11-13T05:38:21,579 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:21,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741838_1019 (size=1264) 2024-11-13T05:38:21,579 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 is not closed yet, will try archiving it next time 2024-11-13T05:38:21,590 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731476281151/Put/vlen=218/seqid=0] 2024-11-13T05:38:21,591 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731476290809/Put/vlen=1045/seqid=0] 2024-11-13T05:38:21,591 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476280148 2024-11-13T05:38:21,591 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:21,591 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:21,591 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 after 0ms 2024-11-13T05:38:21,591 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:21,595 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731476294175/Put/vlen=1045/seqid=0] 2024-11-13T05:38:21,595 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731476296192/Put/vlen=1045/seqid=0] 2024-11-13T05:38:21,595 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 2024-11-13T05:38:21,595 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 2024-11-13T05:38:21,595 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 2024-11-13T05:38:21,596 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 after 1ms 2024-11-13T05:38:21,596 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476299561 2024-11-13T05:38:21,599 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731476299560/Put/vlen=1045/seqid=0] 2024-11-13T05:38:21,599 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:21,599 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:21,599 WARN [IPC Server handler 4 on default port 43693 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-11-13T05:38:21,600 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 after 1ms 2024-11-13T05:38:21,898 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:21,898 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:22,106 WARN [ResponseProcessor for block BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:22,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1745605868_22 at /127.0.0.1:48502 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:33569:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48502 dst: /127.0.0.1:33569 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33569 remote=/127.0.0.1:48502]. Total timeout mills is 60000, 59470 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:22,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1745605868_22 at /127.0.0.1:56146 [Receiving block BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35013:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56146 dst: /127.0.0.1:35013 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:22,106 WARN [DataStreamer for file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 block BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33569,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK], DatanodeInfoWithStorage[127.0.0.1:35013,DS-ed3c7697-f4fb-42f7-8f00-c6ae54344d9b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33569,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]) is bad. 2024-11-13T05:38:22,107 WARN [DataStreamer for file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 block BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:22,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741839_1022 (size=85) 2024-11-13T05:38:22,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741839_1022 (size=85) 2024-11-13T05:38:22,899 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:22,899 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:23,571 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476294175 after 4002ms 2024-11-13T05:38:23,899 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:23,899 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:24,901 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:24,901 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:25,600 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 after 4001ms 2024-11-13T05:38:25,601 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:25,605 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:25,606 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-11-13T05:38:25,606 ERROR [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,606 WARN [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,606 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C39593%2C1731476279539.meta:.meta(num 1731476280569) roll requested 2024-11-13T05:38:25,606 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.meta.1731476305606.meta 2024-11-13T05:38:25,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,616 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476305606.meta 2024-11-13T05:38:25,616 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,617 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,617 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta 2024-11-13T05:38:25,617 WARN [IPC Server handler 0 on default port 43693 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013 2024-11-13T05:38:25,617 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta after 0ms 2024-11-13T05:38:25,620 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38971:38971),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T05:38:25,620 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta is not closed yet, will try archiving it next time 2024-11-13T05:38:25,645 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/info/d2ca4ff61af94d36b64ba3782b188993 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1./info:regioninfo/1731476281156/Put/seqid=0 2024-11-13T05:38:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741841_1025 (size=7125) 2024-11-13T05:38:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741841_1025 (size=7125) 2024-11-13T05:38:25,657 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/info/d2ca4ff61af94d36b64ba3782b188993 2024-11-13T05:38:25,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/ns/0f21037706aa445b851bae56b78d8bf6 is 43, key is default/ns:d/1731476280658/Put/seqid=0 2024-11-13T05:38:25,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741842_1026 (size=5153) 2024-11-13T05:38:25,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741842_1026 (size=5153) 2024-11-13T05:38:25,685 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/ns/0f21037706aa445b851bae56b78d8bf6 2024-11-13T05:38:25,703 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/table/259cff56390943829a77b0d9aa010ecd is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731476281168/Put/seqid=0 2024-11-13T05:38:25,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741843_1027 (size=5438) 2024-11-13T05:38:25,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741843_1027 (size=5438) 2024-11-13T05:38:25,709 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/table/259cff56390943829a77b0d9aa010ecd 2024-11-13T05:38:25,715 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/info/d2ca4ff61af94d36b64ba3782b188993 as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/info/d2ca4ff61af94d36b64ba3782b188993 2024-11-13T05:38:25,721 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/info/d2ca4ff61af94d36b64ba3782b188993, entries=10, sequenceid=11, filesize=7.0 K 2024-11-13T05:38:25,722 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/ns/0f21037706aa445b851bae56b78d8bf6 as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/ns/0f21037706aa445b851bae56b78d8bf6 2024-11-13T05:38:25,728 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/ns/0f21037706aa445b851bae56b78d8bf6, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T05:38:25,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/.tmp/table/259cff56390943829a77b0d9aa010ecd as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/table/259cff56390943829a77b0d9aa010ecd 2024-11-13T05:38:25,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/table/259cff56390943829a77b0d9aa010ecd, entries=2, sequenceid=11, filesize=5.3 K 2024-11-13T05:38:25,736 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 131ms, sequenceid=11, compaction requested=false 2024-11-13T05:38:25,737 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-13T05:38:25,737 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6870496ed7dea82a64bc12c5235753c1 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-13T05:38:25,737 ERROR [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,738 WARN [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39-prefix:a61c20bd729c,39593,1731476279539 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,738 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C39593%2C1731476279539:(num 1731476301571) roll requested 2024-11-13T05:38:25,738 INFO [regionserver/a61c20bd729c:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C39593%2C1731476279539.1731476305738 2024-11-13T05:38:25,743 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 newFile=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476305738 2024-11-13T05:38:25,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:25,744 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476305738 2024-11-13T05:38:25,744 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,744 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1384789590-172.17.0.3-1731476276781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:25,744 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:25,745 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 after 1ms 2024-11-13T05:38:25,746 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.1731476301571 to hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs/a61c20bd729c%2C39593%2C1731476279539.1731476301571 2024-11-13T05:38:25,746 DEBUG [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:38971:38971)] 2024-11-13T05:38:25,762 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/.tmp/info/87686fe0e01e40268d2f8dea710bc791 is 1080, key is row1002/info:/1731476290809/Put/seqid=0 2024-11-13T05:38:25,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741845_1029 (size=9270) 2024-11-13T05:38:25,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741845_1029 (size=9270) 2024-11-13T05:38:25,768 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/.tmp/info/87686fe0e01e40268d2f8dea710bc791 2024-11-13T05:38:25,774 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/.tmp/info/87686fe0e01e40268d2f8dea710bc791 as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/info/87686fe0e01e40268d2f8dea710bc791 2024-11-13T05:38:25,780 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/info/87686fe0e01e40268d2f8dea710bc791, entries=4, sequenceid=8, filesize=9.1 K 2024-11-13T05:38:25,781 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 6870496ed7dea82a64bc12c5235753c1 in 44ms, sequenceid=8, compaction requested=false 2024-11-13T05:38:25,781 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6870496ed7dea82a64bc12c5235753c1: 2024-11-13T05:38:25,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:38:25,785 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:38:25,786 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:38:25,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:25,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:25,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:38:25,786 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:38:25,786 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1722148593, stopped=false 2024-11-13T05:38:25,786 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,41685,1731476279368 2024-11-13T05:38:25,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:38:25,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:38:25,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:25,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:38:25,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:25,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:38:25,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:38:25,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:25,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:38:25,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:38:25,827 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,39593,1731476279539' ***** 2024-11-13T05:38:25,827 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:38:25,827 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(3091): Received CLOSE for 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,39593,1731476279539 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:39593. 2024-11-13T05:38:25,827 DEBUG [RS:0;a61c20bd729c:39593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:38:25,827 DEBUG [RS:0;a61c20bd729c:39593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:25,827 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6870496ed7dea82a64bc12c5235753c1, disabling compactions & flushes 2024-11-13T05:38:25,827 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:38:25,827 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:25,827 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:38:25,827 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. after waiting 0 ms 2024-11-13T05:38:25,827 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:25,828 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:38:25,828 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6870496ed7dea82a64bc12c5235753c1=TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1.} 2024-11-13T05:38:25,828 DEBUG [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6870496ed7dea82a64bc12c5235753c1 2024-11-13T05:38:25,828 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:38:25,828 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:38:25,828 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:38:25,828 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:38:25,828 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:38:25,831 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T05:38:25,831 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/data/default/TestLogRolling-testLogRollOnPipelineRestart/6870496ed7dea82a64bc12c5235753c1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T05:38:25,832 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:38:25,832 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:38:25,832 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:25,832 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476305828Running coprocessor pre-close hooks at 1731476305828Disabling compacts and flushes for region at 1731476305828Disabling writes for close at 1731476305828Writing region close event to WAL at 1731476305829 (+1 ms)Running coprocessor post-close hooks at 1731476305832 (+3 ms)Closed at 1731476305832 2024-11-13T05:38:25,832 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6870496ed7dea82a64bc12c5235753c1: Waiting for close lock at 1731476305827Running coprocessor pre-close hooks at 1731476305827Disabling compacts and flushes for region at 1731476305827Disabling writes for close at 1731476305827Writing region close event to WAL at 1731476305828 (+1 ms)Running coprocessor post-close hooks at 1731476305832 (+4 ms)Closed at 1731476305832 2024-11-13T05:38:25,832 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:38:25,832 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731476280779.6870496ed7dea82a64bc12c5235753c1. 2024-11-13T05:38:25,903 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:25,903 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:26,017 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:38:26,028 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,39593,1731476279539; all regions closed. 2024-11-13T05:38:26,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:26,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:26,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:26,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:26,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:26,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741840_1023 (size=825) 2024-11-13T05:38:26,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741840_1023 (size=825) 2024-11-13T05:38:26,055 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:38:26,055 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:38:26,904 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:26,904 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:26,936 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-13T05:38:27,904 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:27,904 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:28,107 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-13T05:38:28,905 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:28,905 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:29,352 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:38:29,619 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta after 4002ms 2024-11-13T05:38:29,619 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/WALs/a61c20bd729c,39593,1731476279539/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta to hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs/a61c20bd729c%2C39593%2C1731476279539.meta.1731476280569.meta 2024-11-13T05:38:29,624 DEBUG [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs 2024-11-13T05:38:29,625 INFO [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C39593%2C1731476279539.meta:.meta(num 1731476305606) 2024-11-13T05:38:29,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741844_1028 (size=1162) 2024-11-13T05:38:29,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741844_1028 (size=1162) 2024-11-13T05:38:29,633 DEBUG [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs 2024-11-13T05:38:29,633 INFO [RS:0;a61c20bd729c:39593 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C39593%2C1731476279539:(num 1731476305738) 2024-11-13T05:38:29,633 DEBUG [RS:0;a61c20bd729c:39593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:29,633 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:38:29,633 INFO [RS:0;a61c20bd729c:39593 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:38:29,634 INFO [RS:0;a61c20bd729c:39593 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:38:29,634 INFO [RS:0;a61c20bd729c:39593 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:38:29,634 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:38:29,634 INFO [RS:0;a61c20bd729c:39593 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39593 2024-11-13T05:38:29,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,39593,1731476279539 2024-11-13T05:38:29,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:38:29,657 INFO [RS:0;a61c20bd729c:39593 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:38:29,667 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,39593,1731476279539] 2024-11-13T05:38:29,678 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,39593,1731476279539 already deleted, retry=false 2024-11-13T05:38:29,678 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,39593,1731476279539 expired; onlineServers=0 2024-11-13T05:38:29,678 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,41685,1731476279368' ***** 2024-11-13T05:38:29,678 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:38:29,678 INFO [M:0;a61c20bd729c:41685 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:38:29,679 INFO [M:0;a61c20bd729c:41685 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:38:29,679 DEBUG [M:0;a61c20bd729c:41685 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:38:29,679 DEBUG [M:0;a61c20bd729c:41685 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:38:29,679 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:38:29,679 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476279878 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476279878,5,FailOnTimeoutGroup] 2024-11-13T05:38:29,679 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476279878 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476279878,5,FailOnTimeoutGroup] 2024-11-13T05:38:29,679 INFO [M:0;a61c20bd729c:41685 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:38:29,680 INFO [M:0;a61c20bd729c:41685 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:38:29,680 DEBUG [M:0;a61c20bd729c:41685 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:38:29,680 INFO [M:0;a61c20bd729c:41685 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:38:29,680 INFO [M:0;a61c20bd729c:41685 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:38:29,680 INFO [M:0;a61c20bd729c:41685 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:38:29,680 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:38:29,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:38:29,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:29,689 DEBUG [M:0;a61c20bd729c:41685 {}] zookeeper.ZKUtil(347): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:38:29,689 WARN [M:0;a61c20bd729c:41685 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:38:29,690 INFO [M:0;a61c20bd729c:41685 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/.lastflushedseqids 2024-11-13T05:38:29,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741846_1030 (size=139) 2024-11-13T05:38:29,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741846_1030 (size=139) 2024-11-13T05:38:29,696 INFO [M:0;a61c20bd729c:41685 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:38:29,696 INFO [M:0;a61c20bd729c:41685 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:38:29,696 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:38:29,696 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:29,696 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:29,696 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:38:29,696 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:29,696 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB 2024-11-13T05:38:29,697 ERROR [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData-prefix:a61c20bd729c,41685,1731476279368 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:29,697 WARN [FSHLog-0-hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData-prefix:a61c20bd729c,41685,1731476279368 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:29,697 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog a61c20bd729c%2C41685%2C1731476279368:(num 1731476279670) roll requested 2024-11-13T05:38:29,697 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C41685%2C1731476279368.1731476309697 2024-11-13T05:38:29,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,703 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476309697 2024-11-13T05:38:29,703 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:29,703 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:44119,DS-ad6ada8f-600c-4061-9b0a-648cfa28c04f,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-13T05:38:29,703 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 2024-11-13T05:38:29,703 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38971:38971),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T05:38:29,704 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 is not closed yet, will try archiving it next time 2024-11-13T05:38:29,704 WARN [IPC Server handler 2 on default port 43693 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015 2024-11-13T05:38:29,704 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 after 1ms 2024-11-13T05:38:29,720 DEBUG [M:0;a61c20bd729c:41685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ddc7616680c4ed2b5e44361700dba09 is 82, key is hbase:meta,,1/info:regioninfo/1731476280604/Put/seqid=0 2024-11-13T05:38:29,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741848_1033 (size=5672) 2024-11-13T05:38:29,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741848_1033 (size=5672) 2024-11-13T05:38:29,725 INFO [M:0;a61c20bd729c:41685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ddc7616680c4ed2b5e44361700dba09 2024-11-13T05:38:29,743 DEBUG [M:0;a61c20bd729c:41685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f58bc92719c240fe872db8b31f5d4ffc is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731476281173/Put/seqid=0 2024-11-13T05:38:29,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741849_1034 (size=6118) 2024-11-13T05:38:29,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741849_1034 (size=6118) 2024-11-13T05:38:29,748 INFO [M:0;a61c20bd729c:41685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f58bc92719c240fe872db8b31f5d4ffc 2024-11-13T05:38:29,766 DEBUG [M:0;a61c20bd729c:41685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e598cf21943a4decb6eea067cf153b29 is 69, key is a61c20bd729c,39593,1731476279539/rs:state/1731476279992/Put/seqid=0 2024-11-13T05:38:29,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:38:29,768 INFO [RS:0;a61c20bd729c:39593 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:38:29,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39593-0x10132b34be50001, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:38:29,768 INFO [RS:0;a61c20bd729c:39593 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,39593,1731476279539; zookeeper connection closed. 2024-11-13T05:38:29,768 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2eaa9b55 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2eaa9b55 2024-11-13T05:38:29,768 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:38:29,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741850_1035 (size=5156) 2024-11-13T05:38:29,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741850_1035 (size=5156) 2024-11-13T05:38:29,771 INFO [M:0;a61c20bd729c:41685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e598cf21943a4decb6eea067cf153b29 2024-11-13T05:38:29,797 DEBUG [M:0;a61c20bd729c:41685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/72d03178c085456dae8b0997807ba8b7 is 52, key is load_balancer_on/state:d/1731476280774/Put/seqid=0 2024-11-13T05:38:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741851_1036 (size=5056) 2024-11-13T05:38:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741851_1036 (size=5056) 2024-11-13T05:38:29,802 INFO [M:0;a61c20bd729c:41685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/72d03178c085456dae8b0997807ba8b7 2024-11-13T05:38:29,808 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1ddc7616680c4ed2b5e44361700dba09 as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1ddc7616680c4ed2b5e44361700dba09 2024-11-13T05:38:29,814 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1ddc7616680c4ed2b5e44361700dba09, entries=8, sequenceid=56, filesize=5.5 K 2024-11-13T05:38:29,815 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f58bc92719c240fe872db8b31f5d4ffc as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f58bc92719c240fe872db8b31f5d4ffc 2024-11-13T05:38:29,820 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f58bc92719c240fe872db8b31f5d4ffc, entries=6, sequenceid=56, filesize=6.0 K 2024-11-13T05:38:29,821 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e598cf21943a4decb6eea067cf153b29 as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e598cf21943a4decb6eea067cf153b29 2024-11-13T05:38:29,826 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e598cf21943a4decb6eea067cf153b29, entries=1, sequenceid=56, filesize=5.0 K 2024-11-13T05:38:29,827 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/72d03178c085456dae8b0997807ba8b7 as hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/72d03178c085456dae8b0997807ba8b7 2024-11-13T05:38:29,831 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/72d03178c085456dae8b0997807ba8b7, entries=1, sequenceid=56, filesize=4.9 K 2024-11-13T05:38:29,832 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=56, compaction requested=false 2024-11-13T05:38:29,834 INFO [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:29,834 DEBUG [M:0;a61c20bd729c:41685 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476309696Disabling compacts and flushes for region at 1731476309696Disabling writes for close at 1731476309696Obtaining lock to block concurrent updates at 1731476309696Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476309696Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731476309697 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476309704 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476309704Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476309719 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476309719Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476309730 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476309743 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476309743Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476309752 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476309766 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476309766Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476309776 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476309796 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476309797 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b0c39b: reopening flushed file at 1731476309807 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ce36176: reopening flushed file at 1731476309814 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67576b8b: reopening flushed file at 1731476309820 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ba0deb9: reopening flushed file at 1731476309826 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=56, compaction requested=false at 1731476309832 (+6 ms)Writing region close event to WAL at 1731476309834 (+2 ms)Closed at 1731476309834 2024-11-13T05:38:29,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:38:29,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33569 is added to blk_1073741847_1031 (size=757) 2024-11-13T05:38:29,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35013 is added to blk_1073741847_1031 (size=757) 2024-11-13T05:38:29,906 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:29,906 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:30,833 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,833 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,858 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,858 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,858 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,860 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,865 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,865 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:30,907 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:30,907 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:31,105 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-13T05:38:31,368 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:38:31,369 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,369 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,369 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,370 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,391 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,391 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,391 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,392 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,392 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,392 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,397 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,397 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,397 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,401 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:31,907 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:31,907 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:32,908 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:32,908 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:33,705 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 after 4002ms 2024-11-13T05:38:33,705 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/WALs/a61c20bd729c,41685,1731476279368/a61c20bd729c%2C41685%2C1731476279368.1731476279670 to hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/oldWALs/a61c20bd729c%2C41685%2C1731476279368.1731476279670 2024-11-13T05:38:33,707 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/MasterData/oldWALs/a61c20bd729c%2C41685%2C1731476279368.1731476279670 to hdfs://localhost:43693/user/jenkins/test-data/88444495-64de-d60b-4b0b-f2ae1561cf39/oldWALs/a61c20bd729c%2C41685%2C1731476279368.1731476279670$masterlocalwal$ 2024-11-13T05:38:33,707 INFO [M:0;a61c20bd729c:41685 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:38:33,707 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:38:33,708 INFO [M:0;a61c20bd729c:41685 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41685 2024-11-13T05:38:33,708 INFO [M:0;a61c20bd729c:41685 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:38:33,874 INFO [M:0;a61c20bd729c:41685 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:38:33,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:38:33,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41685-0x10132b34be50000, quorum=127.0.0.1:57910, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:38:33,879 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55b58292{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:33,880 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@90a8632{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:33,880 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:33,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@324d7520{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:33,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2cd6e960{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:33,881 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:38:33,881 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:38:33,881 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1384789590-172.17.0.3-1731476276781 (Datanode Uuid 500baa8a-43de-463b-974c-f32c261fb587) service to localhost/127.0.0.1:43693 2024-11-13T05:38:33,881 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:38:33,882 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data3/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:33,882 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data4/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:33,882 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:38:33,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d04913a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:33,885 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@263c339e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:33,885 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:33,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6edb8d2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:33,885 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47ddff64{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:33,886 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:38:33,886 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:38:33,886 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:38:33,886 WARN [BP-1384789590-172.17.0.3-1731476276781 heartbeating to localhost/127.0.0.1:43693 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1384789590-172.17.0.3-1731476276781 (Datanode Uuid ac95cf31-264e-4108-9356-b85ca1914e9f) service to localhost/127.0.0.1:43693 2024-11-13T05:38:33,887 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data1/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:33,887 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/cluster_ed4a0c6c-d1be-ba3b-0d23-f02de40db6b2/data/data2/current/BP-1384789590-172.17.0.3-1731476276781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:38:33,887 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:38:33,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@424e904a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:38:33,894 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d7adad9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:38:33,894 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:38:33,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30acbc29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:38:33,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69a0d4c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir/,STOPPED} 2024-11-13T05:38:33,901 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:38:33,909 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:33,909 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:33,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:38:33,925 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43693 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:43693 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:43693 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43693 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:43693 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43693 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:43693 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43693 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 221), ProcessCount=11 (was 11), AvailableMemoryMB=3105 (was 3327) 2024-11-13T05:38:33,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=3105 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.log.dir so I do NOT create it in target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/888a5f63-704b-5ab2-13f7-7850e32edc50/hadoop.tmp.dir so I do NOT create it in target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4, deleteOnExit=true 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/test.cache.data in system properties and HBase conf 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:38:33,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:38:33,933 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:38:33,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:38:33,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:38:33,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:38:33,947 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:38:34,250 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:34,256 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:34,257 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:34,257 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:34,257 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:38:34,258 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:34,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@399a62ef{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:34,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48e6714f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:34,355 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54b46454{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/java.io.tmpdir/jetty-localhost-36871-hadoop-hdfs-3_4_1-tests_jar-_-any-1498889963234096038/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:38:34,356 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c2a0d6c{HTTP/1.1, (http/1.1)}{localhost:36871} 2024-11-13T05:38:34,356 INFO [Time-limited test {}] server.Server(415): Started @196324ms 2024-11-13T05:38:34,367 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:38:34,618 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:34,621 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:34,621 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:34,621 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:34,622 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:38:34,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@80048ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:34,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@724218ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:34,719 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@166de790{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/java.io.tmpdir/jetty-localhost-40635-hadoop-hdfs-3_4_1-tests_jar-_-any-9556857376541970085/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:34,719 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26ab0093{HTTP/1.1, (http/1.1)}{localhost:40635} 2024-11-13T05:38:34,719 INFO [Time-limited test {}] server.Server(415): Started @196687ms 2024-11-13T05:38:34,720 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:38:34,750 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:38:34,753 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:38:34,753 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:38:34,753 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:38:34,753 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:38:34,754 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c103cf7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:38:34,754 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ba1dfd1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:38:34,848 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3967996f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/java.io.tmpdir/jetty-localhost-42587-hadoop-hdfs-3_4_1-tests_jar-_-any-13744329996627857121/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:38:34,848 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b676e42{HTTP/1.1, (http/1.1)}{localhost:42587} 2024-11-13T05:38:34,848 INFO [Time-limited test {}] server.Server(415): Started @196816ms 2024-11-13T05:38:34,849 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:38:34,909 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:34,909 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:34,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:38:34,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:38:34,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:38:34,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-13T05:38:35,910 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:35,910 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:35,966 WARN [Thread-1652 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data1/current/BP-1057966739-172.17.0.3-1731476313957/current, will proceed with Du for space computation calculation, 2024-11-13T05:38:35,966 WARN [Thread-1653 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data2/current/BP-1057966739-172.17.0.3-1731476313957/current, will proceed with Du for space computation calculation, 2024-11-13T05:38:35,988 WARN [Thread-1616 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:38:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1eb324ce71182cdd with lease ID 0x5e9f4603cf99b5b3: Processing first storage report for DS-0d7bf4be-6669-41c0-af72-dacc5be51c79 from datanode DatanodeRegistration(127.0.0.1:39715, datanodeUuid=5e2c7fb6-cedc-4c11-969b-49535578e07e, infoPort=41159, infoSecurePort=0, ipcPort=35393, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957) 2024-11-13T05:38:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1eb324ce71182cdd with lease ID 0x5e9f4603cf99b5b3: from storage DS-0d7bf4be-6669-41c0-af72-dacc5be51c79 node DatanodeRegistration(127.0.0.1:39715, datanodeUuid=5e2c7fb6-cedc-4c11-969b-49535578e07e, infoPort=41159, infoSecurePort=0, ipcPort=35393, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1eb324ce71182cdd with lease ID 0x5e9f4603cf99b5b3: Processing first storage report for DS-3e3ba0d5-9f40-42a4-bb12-ad04844d5391 from datanode DatanodeRegistration(127.0.0.1:39715, datanodeUuid=5e2c7fb6-cedc-4c11-969b-49535578e07e, infoPort=41159, infoSecurePort=0, ipcPort=35393, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957) 2024-11-13T05:38:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1eb324ce71182cdd with lease ID 0x5e9f4603cf99b5b3: from storage DS-3e3ba0d5-9f40-42a4-bb12-ad04844d5391 node DatanodeRegistration(127.0.0.1:39715, datanodeUuid=5e2c7fb6-cedc-4c11-969b-49535578e07e, infoPort=41159, infoSecurePort=0, ipcPort=35393, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:36,070 WARN [Thread-1663 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data3/current/BP-1057966739-172.17.0.3-1731476313957/current, will proceed with Du for space computation calculation, 2024-11-13T05:38:36,070 WARN [Thread-1664 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data4/current/BP-1057966739-172.17.0.3-1731476313957/current, will proceed with Du for space computation calculation, 2024-11-13T05:38:36,084 WARN [Thread-1639 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:38:36,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c946fb1690b30ca with lease ID 0x5e9f4603cf99b5b4: Processing first storage report for DS-07d89bf3-a7f3-423c-8380-0fab45bc3798 from datanode DatanodeRegistration(127.0.0.1:38459, datanodeUuid=a46da1c1-de96-42ef-934b-c2e13f618226, infoPort=43547, infoSecurePort=0, ipcPort=34741, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957) 2024-11-13T05:38:36,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c946fb1690b30ca with lease ID 0x5e9f4603cf99b5b4: from storage DS-07d89bf3-a7f3-423c-8380-0fab45bc3798 node DatanodeRegistration(127.0.0.1:38459, datanodeUuid=a46da1c1-de96-42ef-934b-c2e13f618226, infoPort=43547, infoSecurePort=0, ipcPort=34741, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:36,087 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c946fb1690b30ca with lease ID 0x5e9f4603cf99b5b4: Processing first storage report for DS-3eeeee4b-83a0-4ef4-887d-99386d81dc6e from datanode DatanodeRegistration(127.0.0.1:38459, datanodeUuid=a46da1c1-de96-42ef-934b-c2e13f618226, infoPort=43547, infoSecurePort=0, ipcPort=34741, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957) 2024-11-13T05:38:36,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c946fb1690b30ca with lease ID 0x5e9f4603cf99b5b4: from storage DS-3eeeee4b-83a0-4ef4-887d-99386d81dc6e node DatanodeRegistration(127.0.0.1:38459, datanodeUuid=a46da1c1-de96-42ef-934b-c2e13f618226, infoPort=43547, infoSecurePort=0, ipcPort=34741, storageInfo=lv=-57;cid=testClusterID;nsid=347992052;c=1731476313957), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:38:36,180 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07 2024-11-13T05:38:36,183 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/zookeeper_0, clientPort=55116, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:38:36,184 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55116 2024-11-13T05:38:36,184 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,185 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:38:36,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:38:36,199 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c with version=8 2024-11-13T05:38:36,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:38:36,201 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:38:36,201 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:38:36,202 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46121 2024-11-13T05:38:36,203 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46121 connecting to ZooKeeper ensemble=127.0.0.1:55116 2024-11-13T05:38:36,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:461210x0, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:38:36,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46121-0x10132b3dbc20000 connected 2024-11-13T05:38:36,341 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:38:36,344 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c, hbase.cluster.distributed=false 2024-11-13T05:38:36,347 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:38:36,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46121 2024-11-13T05:38:36,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46121 2024-11-13T05:38:36,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46121 2024-11-13T05:38:36,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46121 2024-11-13T05:38:36,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46121 2024-11-13T05:38:36,365 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:38:36,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:38:36,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35365 2024-11-13T05:38:36,367 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35365 connecting to ZooKeeper ensemble=127.0.0.1:55116 2024-11-13T05:38:36,367 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,368 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353650x0, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:38:36,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:353650x0, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:38:36,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35365-0x10132b3dbc20001 connected 2024-11-13T05:38:36,384 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:38:36,384 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:38:36,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:38:36,387 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:38:36,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35365 2024-11-13T05:38:36,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35365 2024-11-13T05:38:36,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35365 2024-11-13T05:38:36,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35365 2024-11-13T05:38:36,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35365 2024-11-13T05:38:36,404 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:46121 2024-11-13T05:38:36,404 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:38:36,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:38:36,415 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:38:36,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,425 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:38:36,426 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,46121,1731476316201 from backup master directory 2024-11-13T05:38:36,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:38:36,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:38:36,436 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:38:36,436 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,442 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/hbase.id] with ID: 9aaeb61b-74f1-4088-b8bb-714690c917d6 2024-11-13T05:38:36,442 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/.tmp/hbase.id 2024-11-13T05:38:36,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:38:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:38:36,454 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/.tmp/hbase.id]:[hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/hbase.id] 2024-11-13T05:38:36,466 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:36,466 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:38:36,468 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-13T05:38:36,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:38:36,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:38:36,486 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:38:36,487 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:38:36,487 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:38:36,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:38:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:38:36,495 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store 2024-11-13T05:38:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:38:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:38:36,504 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:36,504 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:38:36,504 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:36,504 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:36,504 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:38:36,504 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:36,504 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:38:36,504 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476316504Disabling compacts and flushes for region at 1731476316504Disabling writes for close at 1731476316504Writing region close event to WAL at 1731476316504Closed at 1731476316504 2024-11-13T05:38:36,505 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/.initializing 2024-11-13T05:38:36,505 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/WALs/a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,507 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C46121%2C1731476316201, suffix=, logDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/WALs/a61c20bd729c,46121,1731476316201, archiveDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/oldWALs, maxLogs=10 2024-11-13T05:38:36,507 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C46121%2C1731476316201.1731476316507 2024-11-13T05:38:36,511 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/WALs/a61c20bd729c,46121,1731476316201/a61c20bd729c%2C46121%2C1731476316201.1731476316507 2024-11-13T05:38:36,512 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43547:43547),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T05:38:36,512 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:38:36,513 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:36,513 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,513 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:38:36,516 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:36,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:38:36,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:38:36,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:38:36,519 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:38:36,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:38:36,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:38:36,521 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,522 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,522 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,523 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,523 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,524 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:38:36,525 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:38:36,527 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:38:36,527 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=724777, jitterRate=-0.07839891314506531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:38:36,527 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476316513Initializing all the Stores at 1731476316514 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476316514Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476316514Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476316514Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476316514Cleaning up temporary data from old regions at 1731476316523 (+9 ms)Region opened successfully at 1731476316527 (+4 ms) 2024-11-13T05:38:36,528 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:38:36,530 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff37f51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:38:36,531 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:38:36,531 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:38:36,531 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:38:36,531 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:38:36,532 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:38:36,532 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:38:36,532 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:38:36,534 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:38:36,534 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:38:36,541 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:38:36,541 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:38:36,542 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:38:36,551 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:38:36,552 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:38:36,553 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:38:36,562 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:38:36,563 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:38:36,572 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:38:36,577 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:38:36,583 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:38:36,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:38:36,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:38:36,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,594 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,46121,1731476316201, sessionid=0x10132b3dbc20000, setting cluster-up flag (Was=false) 2024-11-13T05:38:36,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,646 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:38:36,647 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:36,899 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:38:36,902 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,46121,1731476316201 2024-11-13T05:38:36,904 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:38:36,907 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:38:36,908 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:38:36,908 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:38:36,908 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,46121,1731476316201 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:38:36,911 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:36,911 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:38:36,912 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:36,913 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476346913 2024-11-13T05:38:36,913 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:36,914 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:38:36,914 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:38:36,914 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:38:36,915 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:38:36,915 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:38:36,915 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476316915,5,FailOnTimeoutGroup] 2024-11-13T05:38:36,915 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,915 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476316915,5,FailOnTimeoutGroup] 2024-11-13T05:38:36,915 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:36,915 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:38:36,915 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:36,915 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:36,915 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:38:36,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:38:36,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:38:36,924 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:38:36,924 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c 2024-11-13T05:38:36,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:38:36,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:38:36,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:36,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:38:36,933 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:38:36,933 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:36,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:38:36,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:38:36,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:36,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:38:36,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:38:36,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:36,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:38:36,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:38:36,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:36,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:36,940 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:38:36,940 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740 2024-11-13T05:38:36,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740 2024-11-13T05:38:36,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:38:36,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:38:36,942 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:38:36,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:38:36,945 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:38:36,945 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=853296, jitterRate=0.08502276241779327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:38:36,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476316931Initializing all the Stores at 1731476316932 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476316932Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476316932Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476316932Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476316932Cleaning up temporary data from old regions at 1731476316942 (+10 ms)Region opened successfully at 1731476316946 (+4 ms) 2024-11-13T05:38:36,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:38:36,946 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:38:36,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:38:36,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:38:36,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:38:36,946 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:38:36,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476316946Disabling compacts and flushes for region at 1731476316946Disabling writes for close at 1731476316946Writing region close event to WAL at 1731476316946Closed at 1731476316946 2024-11-13T05:38:36,947 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:38:36,947 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:38:36,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:38:36,949 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:38:36,950 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:38:36,993 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(746): ClusterId : 9aaeb61b-74f1-4088-b8bb-714690c917d6 2024-11-13T05:38:36,993 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:38:37,005 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:38:37,005 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:38:37,015 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:38:37,015 DEBUG [RS:0;a61c20bd729c:35365 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169dc55c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:38:37,025 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:35365 2024-11-13T05:38:37,026 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:38:37,026 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:38:37,026 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:38:37,026 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,46121,1731476316201 with port=35365, startcode=1731476316365 2024-11-13T05:38:37,027 DEBUG [RS:0;a61c20bd729c:35365 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:38:37,029 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47013, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:38:37,029 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46121 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,029 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46121 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,031 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c 2024-11-13T05:38:37,031 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40757 2024-11-13T05:38:37,031 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:38:37,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:38:37,036 DEBUG [RS:0;a61c20bd729c:35365 {}] zookeeper.ZKUtil(111): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,036 WARN [RS:0;a61c20bd729c:35365 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:38:37,036 INFO [RS:0;a61c20bd729c:35365 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:38:37,036 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,036 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,35365,1731476316365] 2024-11-13T05:38:37,039 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:38:37,041 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:38:37,041 INFO [RS:0;a61c20bd729c:35365 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:38:37,041 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,041 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:38:37,042 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:38:37,042 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:38:37,042 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:38:37,043 DEBUG [RS:0;a61c20bd729c:35365 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:38:37,044 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,045 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,045 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,045 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,045 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,045 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35365,1731476316365-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:38:37,062 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:38:37,063 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35365,1731476316365-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,063 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,063 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.Replication(171): a61c20bd729c,35365,1731476316365 started 2024-11-13T05:38:37,076 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,076 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,35365,1731476316365, RpcServer on a61c20bd729c/172.17.0.3:35365, sessionid=0x10132b3dbc20001 2024-11-13T05:38:37,076 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:38:37,076 DEBUG [RS:0;a61c20bd729c:35365 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,076 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,35365,1731476316365' 2024-11-13T05:38:37,076 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:38:37,077 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:38:37,077 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:38:37,077 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:38:37,077 DEBUG [RS:0;a61c20bd729c:35365 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,077 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,35365,1731476316365' 2024-11-13T05:38:37,077 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:38:37,078 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:38:37,078 DEBUG [RS:0;a61c20bd729c:35365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:38:37,078 INFO [RS:0;a61c20bd729c:35365 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:38:37,078 INFO [RS:0;a61c20bd729c:35365 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:38:37,100 WARN [a61c20bd729c:46121 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:38:37,180 INFO [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C35365%2C1731476316365, suffix=, logDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365, archiveDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/oldWALs, maxLogs=32 2024-11-13T05:38:37,181 INFO [RS:0;a61c20bd729c:35365 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35365%2C1731476316365.1731476317181 2024-11-13T05:38:37,187 INFO [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476317181 2024-11-13T05:38:37,188 DEBUG [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:43547:43547)] 2024-11-13T05:38:37,350 DEBUG [a61c20bd729c:46121 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:38:37,352 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,356 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,35365,1731476316365, state=OPENING 2024-11-13T05:38:37,383 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:38:37,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:37,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:38:37,395 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:38:37,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,35365,1731476316365}] 2024-11-13T05:38:37,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:37,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:37,552 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:38:37,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:38:37,562 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:38:37,562 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:38:37,565 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C35365%2C1731476316365.meta, suffix=.meta, logDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365, archiveDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/oldWALs, maxLogs=32 2024-11-13T05:38:37,565 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35365%2C1731476316365.meta.1731476317565.meta 2024-11-13T05:38:37,572 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.meta.1731476317565.meta 2024-11-13T05:38:37,573 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43547:43547),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T05:38:37,573 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:38:37,574 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:38:37,574 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:38:37,574 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:38:37,574 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:38:37,574 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:37,574 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:38:37,574 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:38:37,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:38:37,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:38:37,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:37,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:37,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:38:37,577 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:38:37,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:37,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:37,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:38:37,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:38:37,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:37,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:37,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:38:37,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:38:37,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:37,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:38:37,580 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:38:37,581 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740 2024-11-13T05:38:37,582 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740 2024-11-13T05:38:37,583 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:38:37,583 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:38:37,583 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:38:37,585 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:38:37,585 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=833991, jitterRate=0.0604756623506546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:38:37,586 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:38:37,586 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476317574Writing region info on filesystem at 1731476317574Initializing all the Stores at 1731476317575 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476317575Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476317575Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476317575Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476317575Cleaning up temporary data from old regions at 1731476317583 (+8 ms)Running coprocessor post-open hooks at 1731476317586 (+3 ms)Region opened successfully at 1731476317586 2024-11-13T05:38:37,587 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476317552 2024-11-13T05:38:37,589 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:38:37,590 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:38:37,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,591 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,35365,1731476316365, state=OPEN 2024-11-13T05:38:37,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:38:37,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:38:37,637 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:37,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:38:37,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:38:37,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,35365,1731476316365 in 241 msec 2024-11-13T05:38:37,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:38:37,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 692 msec 2024-11-13T05:38:37,644 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:38:37,644 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:38:37,645 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:38:37,645 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,35365,1731476316365, seqNum=-1] 2024-11-13T05:38:37,646 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:38:37,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53689, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:38:37,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 746 msec 2024-11-13T05:38:37,653 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476317653, completionTime=-1 2024-11-13T05:38:37,653 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:38:37,653 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476377655 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476437655 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46121,1731476316201-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46121,1731476316201-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46121,1731476316201-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:46121, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,655 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,657 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:38:37,659 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.223sec 2024-11-13T05:38:37,659 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:38:37,659 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:38:37,659 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:38:37,659 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:38:37,660 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:38:37,660 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46121,1731476316201-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:38:37,660 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46121,1731476316201-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:38:37,662 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:38:37,662 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:38:37,662 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,46121,1731476316201-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:38:37,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44cb90e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:38:37,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,46121,-1 for getting cluster id 2024-11-13T05:38:37,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:38:37,695 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9aaeb61b-74f1-4088-b8bb-714690c917d6' 2024-11-13T05:38:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:38:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9aaeb61b-74f1-4088-b8bb-714690c917d6" 2024-11-13T05:38:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b95d1be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:38:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,46121,-1] 2024-11-13T05:38:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:38:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:38:37,697 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:38:37,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3553982e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:38:37,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:38:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,35365,1731476316365, seqNum=-1] 2024-11-13T05:38:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:38:37,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:38:37,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,46121,1731476316201 2024-11-13T05:38:37,703 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:38:37,706 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:38:37,706 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:38:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a61c20bd729c,46121,1731476316201 2024-11-13T05:38:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ee66a19 2024-11-13T05:38:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:38:37,708 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:38:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-13T05:38:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-13T05:38:37,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:38:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:37,712 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:38:37,712 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:37,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-13T05:38:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:38:37,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:38:37,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741835_1011 (size=405) 2024-11-13T05:38:37,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741835_1011 (size=405) 2024-11-13T05:38:37,721 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ce3a4eab44d10f169642d9f70eeff47, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c 2024-11-13T05:38:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741836_1012 (size=88) 2024-11-13T05:38:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741836_1012 (size=88) 2024-11-13T05:38:37,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:37,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 0ce3a4eab44d10f169642d9f70eeff47, disabling compactions & flushes 2024-11-13T05:38:37,728 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:37,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:37,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. after waiting 0 ms 2024-11-13T05:38:37,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:37,728 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:37,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ce3a4eab44d10f169642d9f70eeff47: Waiting for close lock at 1731476317728Disabling compacts and flushes for region at 1731476317728Disabling writes for close at 1731476317728Writing region close event to WAL at 1731476317728Closed at 1731476317728 2024-11-13T05:38:37,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:38:37,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731476317729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476317729"}]},"ts":"1731476317729"} 2024-11-13T05:38:37,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:38:37,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:38:37,734 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476317734"}]},"ts":"1731476317734"} 2024-11-13T05:38:37,736 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-13T05:38:37,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=0ce3a4eab44d10f169642d9f70eeff47, ASSIGN}] 2024-11-13T05:38:37,738 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=0ce3a4eab44d10f169642d9f70eeff47, ASSIGN 2024-11-13T05:38:37,739 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=0ce3a4eab44d10f169642d9f70eeff47, ASSIGN; state=OFFLINE, location=a61c20bd729c,35365,1731476316365; forceNewPlan=false, retain=false 2024-11-13T05:38:37,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ce3a4eab44d10f169642d9f70eeff47, regionState=OPENING, regionLocation=a61c20bd729c,35365,1731476316365 2024-11-13T05:38:37,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=0ce3a4eab44d10f169642d9f70eeff47, ASSIGN because future has completed 2024-11-13T05:38:37,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ce3a4eab44d10f169642d9f70eeff47, server=a61c20bd729c,35365,1731476316365}] 2024-11-13T05:38:37,912 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:37,912 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:38,054 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:38,054 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ce3a4eab44d10f169642d9f70eeff47, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:38:38,054 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,054 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:38:38,055 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,055 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,057 INFO [StoreOpener-0ce3a4eab44d10f169642d9f70eeff47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,058 INFO [StoreOpener-0ce3a4eab44d10f169642d9f70eeff47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ce3a4eab44d10f169642d9f70eeff47 columnFamilyName info 2024-11-13T05:38:38,059 DEBUG [StoreOpener-0ce3a4eab44d10f169642d9f70eeff47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:38:38,059 INFO [StoreOpener-0ce3a4eab44d10f169642d9f70eeff47-1 {}] regionserver.HStore(327): Store=0ce3a4eab44d10f169642d9f70eeff47/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:38:38,059 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,061 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,061 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,062 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,062 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,064 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,066 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:38:38,067 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0ce3a4eab44d10f169642d9f70eeff47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709005, jitterRate=-0.09845432639122009}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:38:38,067 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:38:38,067 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0ce3a4eab44d10f169642d9f70eeff47: Running coprocessor pre-open hook at 1731476318055Writing region info on filesystem at 1731476318055Initializing all the Stores at 1731476318056 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476318056Cleaning up temporary data from old regions at 1731476318062 (+6 ms)Running coprocessor post-open hooks at 1731476318067 (+5 ms)Region opened successfully at 1731476318067 2024-11-13T05:38:38,068 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47., pid=6, masterSystemTime=1731476318046 2024-11-13T05:38:38,071 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:38,071 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:38,072 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ce3a4eab44d10f169642d9f70eeff47, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,35365,1731476316365 2024-11-13T05:38:38,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ce3a4eab44d10f169642d9f70eeff47, server=a61c20bd729c,35365,1731476316365 because future has completed 2024-11-13T05:38:38,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:38:38,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ce3a4eab44d10f169642d9f70eeff47, server=a61c20bd729c,35365,1731476316365 in 183 msec 2024-11-13T05:38:38,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:38:38,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=0ce3a4eab44d10f169642d9f70eeff47, ASSIGN in 342 msec 2024-11-13T05:38:38,081 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:38:38,082 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476318081"}]},"ts":"1731476318081"} 2024-11-13T05:38:38,084 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-13T05:38:38,085 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:38:38,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 376 msec 2024-11-13T05:38:38,913 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:38,913 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:39,914 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:39,914 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:40,472 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:38:40,473 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,474 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,474 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,474 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,493 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,493 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,493 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,493 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,494 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,494 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,497 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,497 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,497 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,500 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:38:40,915 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:40,915 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:41,916 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:41,916 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:42,918 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:42,918 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:43,040 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:38:43,040 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-13T05:38:43,918 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:43,918 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:44,919 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:44,919 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:44,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:38:44,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:38:44,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:38:44,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:38:44,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:38:44,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:38:44,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:44,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-13T05:38:45,920 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:45,920 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:46,920 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:46,920 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:38:47,755 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-13T05:38:47,755 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-13T05:38:47,760 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:47,760 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47., hostname=a61c20bd729c,35365,1731476316365, seqNum=2] 2024-11-13T05:38:47,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:47,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:38:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:38:47,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:38:47,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:38:47,922 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:47,922 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:47,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:38:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:47,947 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 0ce3a4eab44d10f169642d9f70eeff47 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-13T05:38:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/abb2d5b1eb9a440d86bf1671d1bc6bc6 is 1080, key is row0001/info:/1731476327768/Put/seqid=0 2024-11-13T05:38:47,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741837_1013 (size=6033) 2024-11-13T05:38:47,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741837_1013 (size=6033) 2024-11-13T05:38:47,969 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/abb2d5b1eb9a440d86bf1671d1bc6bc6 2024-11-13T05:38:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/abb2d5b1eb9a440d86bf1671d1bc6bc6 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/abb2d5b1eb9a440d86bf1671d1bc6bc6 2024-11-13T05:38:47,981 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/abb2d5b1eb9a440d86bf1671d1bc6bc6, entries=1, sequenceid=5, filesize=5.9 K 2024-11-13T05:38:47,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce3a4eab44d10f169642d9f70eeff47 in 35ms, sequenceid=5, compaction requested=false 2024-11-13T05:38:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 0ce3a4eab44d10f169642d9f70eeff47: 2024-11-13T05:38:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T05:38:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T05:38:47,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:38:47,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 205 msec 2024-11-13T05:38:47,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 215 msec 2024-11-13T05:38:48,923 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:48,923 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:49,923 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:49,923 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:50,924 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:50,924 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:51,925 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:51,925 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:52,925 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:52,925 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:53,927 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:53,927 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:54,928 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:54,928 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:55,929 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:55,929 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:56,930 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:56,930 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:56,932 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 after 68072ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:56,932 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta after 68063ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:38:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:38:57,885 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-13T05:38:57,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:38:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:38:57,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:38:57,899 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:38:57,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:38:57,933 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:57,933 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:58,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:38:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:58,056 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 0ce3a4eab44d10f169642d9f70eeff47 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-13T05:38:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/93747ff063eb42b4877f4840297586ab is 1080, key is row0002/info:/1731476337888/Put/seqid=0 2024-11-13T05:38:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741838_1014 (size=6033) 2024-11-13T05:38:58,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741838_1014 (size=6033) 2024-11-13T05:38:58,071 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/93747ff063eb42b4877f4840297586ab 2024-11-13T05:38:58,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/93747ff063eb42b4877f4840297586ab as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/93747ff063eb42b4877f4840297586ab 2024-11-13T05:38:58,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/93747ff063eb42b4877f4840297586ab, entries=1, sequenceid=9, filesize=5.9 K 2024-11-13T05:38:58,087 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce3a4eab44d10f169642d9f70eeff47 in 31ms, sequenceid=9, compaction requested=false 2024-11-13T05:38:58,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 0ce3a4eab44d10f169642d9f70eeff47: 2024-11-13T05:38:58,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:38:58,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T05:38:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T05:38:58,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T05:38:58,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 190 msec 2024-11-13T05:38:58,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 201 msec 2024-11-13T05:38:58,934 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:58,934 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:59,935 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:38:59,935 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:00,937 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:00,937 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:01,938 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:01,938 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:02,939 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:02,939 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:03,940 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:03,940 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:04,942 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:04,942 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:05,943 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:05,943 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:06,180 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:39:06,944 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:06,944 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:07,945 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:07,945 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:07,975 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-13T05:39:07,980 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35365%2C1731476316365.1731476347980 2024-11-13T05:39:07,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:07,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:07,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:07,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:07,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:07,988 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476317181 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476347980 2024-11-13T05:39:07,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43547:43547),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T05:39:07,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476317181 is not closed yet, will try archiving it next time 2024-11-13T05:39:07,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:39:07,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741833_1009 (size=5546) 2024-11-13T05:39:07,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741833_1009 (size=5546) 2024-11-13T05:39:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:39:07,994 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:39:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:07,995 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:39:07,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:39:08,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:08,149 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 0ce3a4eab44d10f169642d9f70eeff47 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-13T05:39:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/2714291ecc3e490c8d8dba2b6712b846 is 1080, key is row0003/info:/1731476347977/Put/seqid=0 2024-11-13T05:39:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741840_1016 (size=6033) 2024-11-13T05:39:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741840_1016 (size=6033) 2024-11-13T05:39:08,157 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/2714291ecc3e490c8d8dba2b6712b846 2024-11-13T05:39:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/2714291ecc3e490c8d8dba2b6712b846 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/2714291ecc3e490c8d8dba2b6712b846 2024-11-13T05:39:08,169 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/2714291ecc3e490c8d8dba2b6712b846, entries=1, sequenceid=13, filesize=5.9 K 2024-11-13T05:39:08,170 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce3a4eab44d10f169642d9f70eeff47 in 22ms, sequenceid=13, compaction requested=true 2024-11-13T05:39:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 0ce3a4eab44d10f169642d9f70eeff47: 2024-11-13T05:39:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T05:39:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T05:39:08,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T05:39:08,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec 2024-11-13T05:39:08,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec 2024-11-13T05:39:08,946 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:08,946 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:09,947 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:09,948 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:10,949 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:10,949 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:11,951 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:11,951 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:12,952 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:12,952 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:13,953 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:13,953 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:14,955 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:14,955 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:15,956 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:15,956 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:16,958 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:16,958 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:17,959 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:17,959 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:18,005 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-13T05:39:18,006 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:18,009 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:18,009 DEBUG [Time-limited test {}] regionserver.HStore(1541): 0ce3a4eab44d10f169642d9f70eeff47/info is initiating minor compaction (all files) 2024-11-13T05:39:18,010 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:39:18,010 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:18,010 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 0ce3a4eab44d10f169642d9f70eeff47/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:18,011 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/abb2d5b1eb9a440d86bf1671d1bc6bc6, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/93747ff063eb42b4877f4840297586ab, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/2714291ecc3e490c8d8dba2b6712b846] into tmpdir=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp, totalSize=17.7 K 2024-11-13T05:39:18,012 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting abb2d5b1eb9a440d86bf1671d1bc6bc6, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731476327768 2024-11-13T05:39:18,013 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 93747ff063eb42b4877f4840297586ab, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731476337888 2024-11-13T05:39:18,014 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2714291ecc3e490c8d8dba2b6712b846, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731476347977 2024-11-13T05:39:18,028 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 0ce3a4eab44d10f169642d9f70eeff47#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:18,029 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/8fce746f40f74b18a53a9aea63bd76f0 is 1080, key is row0001/info:/1731476327768/Put/seqid=0 2024-11-13T05:39:18,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741841_1017 (size=8296) 2024-11-13T05:39:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741841_1017 (size=8296) 2024-11-13T05:39:18,041 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/8fce746f40f74b18a53a9aea63bd76f0 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/8fce746f40f74b18a53a9aea63bd76f0 2024-11-13T05:39:18,044 INFO [master/a61c20bd729c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T05:39:18,044 INFO [master/a61c20bd729c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T05:39:18,047 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ce3a4eab44d10f169642d9f70eeff47/info of 0ce3a4eab44d10f169642d9f70eeff47 into 8fce746f40f74b18a53a9aea63bd76f0(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:18,047 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 0ce3a4eab44d10f169642d9f70eeff47: 2024-11-13T05:39:18,050 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35365%2C1731476316365.1731476358049 2024-11-13T05:39:18,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:18,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:18,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:18,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:18,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:18,056 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476347980 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476358049 2024-11-13T05:39:18,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741839_1015 (size=2520) 2024-11-13T05:39:18,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741839_1015 (size=2520) 2024-11-13T05:39:18,060 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:43547:43547)] 2024-11-13T05:39:18,060 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476317181 to hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/oldWALs/a61c20bd729c%2C35365%2C1731476316365.1731476317181 2024-11-13T05:39:18,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:39:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:39:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:39:18,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:39:18,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:39:18,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:39:18,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:39:18,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:18,218 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 0ce3a4eab44d10f169642d9f70eeff47 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-13T05:39:18,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/db2b9311c4674776b11e18e8ebb2864d is 1080, key is row0000/info:/1731476358048/Put/seqid=0 2024-11-13T05:39:18,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741843_1019 (size=6033) 2024-11-13T05:39:18,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741843_1019 (size=6033) 2024-11-13T05:39:18,233 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/db2b9311c4674776b11e18e8ebb2864d 2024-11-13T05:39:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/db2b9311c4674776b11e18e8ebb2864d as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/db2b9311c4674776b11e18e8ebb2864d 2024-11-13T05:39:18,245 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/db2b9311c4674776b11e18e8ebb2864d, entries=1, sequenceid=18, filesize=5.9 K 2024-11-13T05:39:18,246 INFO [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce3a4eab44d10f169642d9f70eeff47 in 28ms, sequenceid=18, compaction requested=false 2024-11-13T05:39:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 0ce3a4eab44d10f169642d9f70eeff47: 2024-11-13T05:39:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T05:39:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T05:39:18,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T05:39:18,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 184 msec 2024-11-13T05:39:18,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 191 msec 2024-11-13T05:39:18,960 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:18,960 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:19,961 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:19,961 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:20,962 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:20,962 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:21,964 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:21,964 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:22,965 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:22,965 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:23,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0ce3a4eab44d10f169642d9f70eeff47, had cached 0 bytes from a total of 14329 2024-11-13T05:39:23,966 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:23,966 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:24,967 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:24,967 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:25,969 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:25,969 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:26,970 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:26,970 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:27,972 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:27,971 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:39:28,156 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-13T05:39:28,162 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35365%2C1731476316365.1731476368162 2024-11-13T05:39:28,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,172 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476358049 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476368162 2024-11-13T05:39:28,173 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43547:43547),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T05:39:28,173 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476358049 is not closed yet, will try archiving it next time 2024-11-13T05:39:28,173 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/WALs/a61c20bd729c,35365,1731476316365/a61c20bd729c%2C35365%2C1731476316365.1731476347980 to hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/oldWALs/a61c20bd729c%2C35365%2C1731476316365.1731476347980 2024-11-13T05:39:28,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:39:28,174 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:39:28,174 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:39:28,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:28,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:28,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741842_1018 (size=2026) 2024-11-13T05:39:28,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:39:28,175 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:39:28,175 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1601953406, stopped=false 2024-11-13T05:39:28,175 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,46121,1731476316201 2024-11-13T05:39:28,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741842_1018 (size=2026) 2024-11-13T05:39:28,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:39:28,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:39:28,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:28,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:39:28,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:28,245 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:39:28,245 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:39:28,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:28,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:39:28,246 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,35365,1731476316365' ***** 2024-11-13T05:39:28,246 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:39:28,246 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:39:28,247 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:39:28,247 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:39:28,247 INFO [RS:0;a61c20bd729c:35365 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:39:28,247 INFO [RS:0;a61c20bd729c:35365 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:39:28,247 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ce3a4eab44d10f169642d9f70eeff47 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,35365,1731476316365 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:35365. 2024-11-13T05:39:28,248 DEBUG [RS:0;a61c20bd729c:35365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:39:28,248 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ce3a4eab44d10f169642d9f70eeff47, disabling compactions & flushes 2024-11-13T05:39:28,248 DEBUG [RS:0;a61c20bd729c:35365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:28,248 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:28,248 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:39:28,248 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. after waiting 0 ms 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:39:28,248 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:28,248 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:39:28,249 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0ce3a4eab44d10f169642d9f70eeff47 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-13T05:39:28,249 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:39:28,249 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0ce3a4eab44d10f169642d9f70eeff47=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.} 2024-11-13T05:39:28,249 DEBUG [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1351): Waiting on 0ce3a4eab44d10f169642d9f70eeff47, 1588230740 2024-11-13T05:39:28,249 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:39:28,249 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:39:28,249 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:39:28,249 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:39:28,249 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:39:28,249 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-13T05:39:28,254 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/b046bb4c7c9341a098441bccba49933d is 1080, key is row0001/info:/1731476368159/Put/seqid=0 2024-11-13T05:39:28,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741845_1021 (size=6033) 2024-11-13T05:39:28,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741845_1021 (size=6033) 2024-11-13T05:39:28,261 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/b046bb4c7c9341a098441bccba49933d 2024-11-13T05:39:28,267 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/info/6134b27e27fd42c9afcc40e39fad96e9 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47./info:regioninfo/1731476318072/Put/seqid=0 2024-11-13T05:39:28,267 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/.tmp/info/b046bb4c7c9341a098441bccba49933d as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/b046bb4c7c9341a098441bccba49933d 2024-11-13T05:39:28,272 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/b046bb4c7c9341a098441bccba49933d, entries=1, sequenceid=22, filesize=5.9 K 2024-11-13T05:39:28,274 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce3a4eab44d10f169642d9f70eeff47 in 26ms, sequenceid=22, compaction requested=true 2024-11-13T05:39:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741846_1022 (size=7308) 2024-11-13T05:39:28,274 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/abb2d5b1eb9a440d86bf1671d1bc6bc6, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/93747ff063eb42b4877f4840297586ab, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/2714291ecc3e490c8d8dba2b6712b846] to archive 2024-11-13T05:39:28,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741846_1022 (size=7308) 2024-11-13T05:39:28,275 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/info/6134b27e27fd42c9afcc40e39fad96e9 2024-11-13T05:39:28,275 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:39:28,277 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/abb2d5b1eb9a440d86bf1671d1bc6bc6 to hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/abb2d5b1eb9a440d86bf1671d1bc6bc6 2024-11-13T05:39:28,278 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/93747ff063eb42b4877f4840297586ab to hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/93747ff063eb42b4877f4840297586ab 2024-11-13T05:39:28,279 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/2714291ecc3e490c8d8dba2b6712b846 to hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/info/2714291ecc3e490c8d8dba2b6712b846 2024-11-13T05:39:28,279 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a61c20bd729c:46121 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-13T05:39:28,279 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [abb2d5b1eb9a440d86bf1671d1bc6bc6=6033, 93747ff063eb42b4877f4840297586ab=6033, 2714291ecc3e490c8d8dba2b6712b846=6033] 2024-11-13T05:39:28,283 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/0ce3a4eab44d10f169642d9f70eeff47/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-13T05:39:28,284 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:28,284 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ce3a4eab44d10f169642d9f70eeff47: Waiting for close lock at 1731476368248Running coprocessor pre-close hooks at 1731476368248Disabling compacts and flushes for region at 1731476368248Disabling writes for close at 1731476368248Obtaining lock to block concurrent updates at 1731476368249 (+1 ms)Preparing flush snapshotting stores in 0ce3a4eab44d10f169642d9f70eeff47 at 1731476368249Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731476368249Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. at 1731476368250 (+1 ms)Flushing 0ce3a4eab44d10f169642d9f70eeff47/info: creating writer at 1731476368250Flushing 0ce3a4eab44d10f169642d9f70eeff47/info: appending metadata at 1731476368253 (+3 ms)Flushing 0ce3a4eab44d10f169642d9f70eeff47/info: closing flushed file at 1731476368253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a91d83a: reopening flushed file at 1731476368267 (+14 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce3a4eab44d10f169642d9f70eeff47 in 26ms, sequenceid=22, compaction requested=true at 1731476368274 (+7 ms)Writing region close event to WAL at 1731476368280 (+6 ms)Running coprocessor post-close hooks at 1731476368284 (+4 ms)Closed at 1731476368284 2024-11-13T05:39:28,284 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731476317709.0ce3a4eab44d10f169642d9f70eeff47. 2024-11-13T05:39:28,299 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/ns/0cc486766c20416b9561f5113f828d5f is 43, key is default/ns:d/1731476317647/Put/seqid=0 2024-11-13T05:39:28,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741847_1023 (size=5153) 2024-11-13T05:39:28,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741847_1023 (size=5153) 2024-11-13T05:39:28,304 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/ns/0cc486766c20416b9561f5113f828d5f 2024-11-13T05:39:28,322 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/table/3f0a1d0c7c724e9a936dd536b30071d4 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731476318081/Put/seqid=0 2024-11-13T05:39:28,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741848_1024 (size=5508) 2024-11-13T05:39:28,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741848_1024 (size=5508) 2024-11-13T05:39:28,327 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/table/3f0a1d0c7c724e9a936dd536b30071d4 2024-11-13T05:39:28,331 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/info/6134b27e27fd42c9afcc40e39fad96e9 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/info/6134b27e27fd42c9afcc40e39fad96e9 2024-11-13T05:39:28,336 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/info/6134b27e27fd42c9afcc40e39fad96e9, entries=10, sequenceid=11, filesize=7.1 K 2024-11-13T05:39:28,337 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/ns/0cc486766c20416b9561f5113f828d5f as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/ns/0cc486766c20416b9561f5113f828d5f 2024-11-13T05:39:28,343 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/ns/0cc486766c20416b9561f5113f828d5f, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T05:39:28,344 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/.tmp/table/3f0a1d0c7c724e9a936dd536b30071d4 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/table/3f0a1d0c7c724e9a936dd536b30071d4 2024-11-13T05:39:28,349 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/table/3f0a1d0c7c724e9a936dd536b30071d4, entries=2, sequenceid=11, filesize=5.4 K 2024-11-13T05:39:28,350 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=11, compaction requested=false 2024-11-13T05:39:28,355 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T05:39:28,355 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:39:28,355 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:39:28,355 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476368249Running coprocessor pre-close hooks at 1731476368249Disabling compacts and flushes for region at 1731476368249Disabling writes for close at 1731476368249Obtaining lock to block concurrent updates at 1731476368250 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731476368250Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731476368250Flushing stores of hbase:meta,,1.1588230740 at 1731476368251 (+1 ms)Flushing 1588230740/info: creating writer at 1731476368251Flushing 1588230740/info: appending metadata at 1731476368267 (+16 ms)Flushing 1588230740/info: closing flushed file at 1731476368267Flushing 1588230740/ns: creating writer at 1731476368280 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731476368298 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731476368298Flushing 1588230740/table: creating writer at 1731476368309 (+11 ms)Flushing 1588230740/table: appending metadata at 1731476368321 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731476368321Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5124c486: reopening flushed file at 1731476368331 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69404fee: reopening flushed file at 1731476368336 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3019d6c2: reopening flushed file at 1731476368344 (+8 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=11, compaction requested=false at 1731476368350 (+6 ms)Writing region close event to WAL at 1731476368352 (+2 ms)Running coprocessor post-close hooks at 1731476368355 (+3 ms)Closed at 1731476368355 2024-11-13T05:39:28,355 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:39:28,449 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,35365,1731476316365; all regions closed. 2024-11-13T05:39:28,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741834_1010 (size=3306) 2024-11-13T05:39:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741834_1010 (size=3306) 2024-11-13T05:39:28,461 DEBUG [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/oldWALs 2024-11-13T05:39:28,461 INFO [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C35365%2C1731476316365.meta:.meta(num 1731476317565) 2024-11-13T05:39:28,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741844_1020 (size=1252) 2024-11-13T05:39:28,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741844_1020 (size=1252) 2024-11-13T05:39:28,467 DEBUG [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/oldWALs 2024-11-13T05:39:28,467 INFO [RS:0;a61c20bd729c:35365 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C35365%2C1731476316365:(num 1731476368162) 2024-11-13T05:39:28,467 DEBUG [RS:0;a61c20bd729c:35365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:28,467 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:39:28,468 INFO [RS:0;a61c20bd729c:35365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:39:28,468 INFO [RS:0;a61c20bd729c:35365 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:39:28,468 INFO [RS:0;a61c20bd729c:35365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:39:28,468 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:39:28,468 INFO [RS:0;a61c20bd729c:35365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35365 2024-11-13T05:39:28,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:39:28,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,35365,1731476316365 2024-11-13T05:39:28,496 INFO [RS:0;a61c20bd729c:35365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:39:28,497 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,35365,1731476316365] 2024-11-13T05:39:28,517 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,35365,1731476316365 already deleted, retry=false 2024-11-13T05:39:28,517 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,35365,1731476316365 expired; onlineServers=0 2024-11-13T05:39:28,517 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,46121,1731476316201' ***** 2024-11-13T05:39:28,517 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:39:28,517 INFO [M:0;a61c20bd729c:46121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:39:28,517 INFO [M:0;a61c20bd729c:46121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:39:28,518 DEBUG [M:0;a61c20bd729c:46121 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:39:28,518 DEBUG [M:0;a61c20bd729c:46121 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:39:28,518 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:39:28,518 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476316915 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476316915,5,FailOnTimeoutGroup] 2024-11-13T05:39:28,518 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476316915 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476316915,5,FailOnTimeoutGroup] 2024-11-13T05:39:28,518 INFO [M:0;a61c20bd729c:46121 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:39:28,518 INFO [M:0;a61c20bd729c:46121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:39:28,518 DEBUG [M:0;a61c20bd729c:46121 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:39:28,519 INFO [M:0;a61c20bd729c:46121 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:39:28,519 INFO [M:0;a61c20bd729c:46121 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:39:28,519 INFO [M:0;a61c20bd729c:46121 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:39:28,519 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:39:28,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:39:28,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:28,528 DEBUG [M:0;a61c20bd729c:46121 {}] zookeeper.ZKUtil(347): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:39:28,528 WARN [M:0;a61c20bd729c:46121 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:39:28,529 INFO [M:0;a61c20bd729c:46121 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/.lastflushedseqids 2024-11-13T05:39:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741849_1025 (size=130) 2024-11-13T05:39:28,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741849_1025 (size=130) 2024-11-13T05:39:28,541 INFO [M:0;a61c20bd729c:46121 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:39:28,542 INFO [M:0;a61c20bd729c:46121 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:39:28,542 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:39:28,542 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:28,542 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:28,542 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:39:28,542 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:28,542 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.54 KB heapSize=54.89 KB 2024-11-13T05:39:28,561 DEBUG [M:0;a61c20bd729c:46121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbea8ce318ff44fa833fe780747db816 is 82, key is hbase:meta,,1/info:regioninfo/1731476317590/Put/seqid=0 2024-11-13T05:39:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741850_1026 (size=5672) 2024-11-13T05:39:28,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741850_1026 (size=5672) 2024-11-13T05:39:28,566 INFO [M:0;a61c20bd729c:46121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbea8ce318ff44fa833fe780747db816 2024-11-13T05:39:28,588 DEBUG [M:0;a61c20bd729c:46121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/489b61b6e1ef4acf9a1ed7221c4ee4dd is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731476318086/Put/seqid=0 2024-11-13T05:39:28,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741851_1027 (size=7817) 2024-11-13T05:39:28,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741851_1027 (size=7817) 2024-11-13T05:39:28,593 INFO [M:0;a61c20bd729c:46121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.94 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/489b61b6e1ef4acf9a1ed7221c4ee4dd 2024-11-13T05:39:28,598 INFO [M:0;a61c20bd729c:46121 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 489b61b6e1ef4acf9a1ed7221c4ee4dd 2024-11-13T05:39:28,607 INFO [RS:0;a61c20bd729c:35365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:39:28,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:39:28,607 INFO [RS:0;a61c20bd729c:35365 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,35365,1731476316365; zookeeper connection closed. 2024-11-13T05:39:28,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35365-0x10132b3dbc20001, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:39:28,607 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@123f662 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@123f662 2024-11-13T05:39:28,607 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:39:28,611 DEBUG [M:0;a61c20bd729c:46121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0a16f581a074d85888550016ef71dd7 is 69, key is a61c20bd729c,35365,1731476316365/rs:state/1731476317029/Put/seqid=0 2024-11-13T05:39:28,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741852_1028 (size=5156) 2024-11-13T05:39:28,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741852_1028 (size=5156) 2024-11-13T05:39:28,616 INFO [M:0;a61c20bd729c:46121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0a16f581a074d85888550016ef71dd7 2024-11-13T05:39:28,633 DEBUG [M:0;a61c20bd729c:46121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7b4aab3fe9794849ba50b012b50b0dc8 is 52, key is load_balancer_on/state:d/1731476317705/Put/seqid=0 2024-11-13T05:39:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741853_1029 (size=5056) 2024-11-13T05:39:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741853_1029 (size=5056) 2024-11-13T05:39:28,638 INFO [M:0;a61c20bd729c:46121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7b4aab3fe9794849ba50b012b50b0dc8 2024-11-13T05:39:28,643 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbea8ce318ff44fa833fe780747db816 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbea8ce318ff44fa833fe780747db816 2024-11-13T05:39:28,648 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbea8ce318ff44fa833fe780747db816, entries=8, sequenceid=121, filesize=5.5 K 2024-11-13T05:39:28,649 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/489b61b6e1ef4acf9a1ed7221c4ee4dd as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/489b61b6e1ef4acf9a1ed7221c4ee4dd 2024-11-13T05:39:28,653 INFO [M:0;a61c20bd729c:46121 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 489b61b6e1ef4acf9a1ed7221c4ee4dd 2024-11-13T05:39:28,654 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/489b61b6e1ef4acf9a1ed7221c4ee4dd, entries=14, sequenceid=121, filesize=7.6 K 2024-11-13T05:39:28,655 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0a16f581a074d85888550016ef71dd7 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0a16f581a074d85888550016ef71dd7 2024-11-13T05:39:28,659 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0a16f581a074d85888550016ef71dd7, entries=1, sequenceid=121, filesize=5.0 K 2024-11-13T05:39:28,661 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7b4aab3fe9794849ba50b012b50b0dc8 as hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7b4aab3fe9794849ba50b012b50b0dc8 2024-11-13T05:39:28,667 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40757/user/jenkins/test-data/ec720194-fdcb-c852-36de-4d5f1a2ad85c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7b4aab3fe9794849ba50b012b50b0dc8, entries=1, sequenceid=121, filesize=4.9 K 2024-11-13T05:39:28,668 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.54 KB/44584, heapSize ~54.83 KB/56144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=121, compaction requested=false 2024-11-13T05:39:28,670 INFO [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:28,670 DEBUG [M:0;a61c20bd729c:46121 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476368542Disabling compacts and flushes for region at 1731476368542Disabling writes for close at 1731476368542Obtaining lock to block concurrent updates at 1731476368542Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476368542Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44584, getHeapSize=56144, getOffHeapSize=0, getCellsCount=140 at 1731476368543 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476368544 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476368544Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476368561 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476368561Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476368570 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476368587 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476368588 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476368598 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476368611 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476368611Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476368620 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476368633 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476368633Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@784d4b89: reopening flushed file at 1731476368642 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15b7ba94: reopening flushed file at 1731476368648 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c0e4c8: reopening flushed file at 1731476368654 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ca45340: reopening flushed file at 1731476368660 (+6 ms)Finished flush of dataSize ~43.54 KB/44584, heapSize ~54.83 KB/56144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=121, compaction requested=false at 1731476368668 (+8 ms)Writing region close event to WAL at 1731476368669 (+1 ms)Closed at 1731476368669 2024-11-13T05:39:28,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:39:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39715 is added to blk_1073741830_1006 (size=52981) 2024-11-13T05:39:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38459 is added to blk_1073741830_1006 (size=52981) 2024-11-13T05:39:28,673 INFO [M:0;a61c20bd729c:46121 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:39:28,673 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:39:28,673 INFO [M:0;a61c20bd729c:46121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46121 2024-11-13T05:39:28,673 INFO [M:0;a61c20bd729c:46121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:39:28,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:39:28,786 INFO [M:0;a61c20bd729c:46121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:39:28,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46121-0x10132b3dbc20000, quorum=127.0.0.1:55116, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:39:28,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3967996f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:39:28,831 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b676e42{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:39:28,831 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:39:28,831 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ba1dfd1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:39:28,831 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c103cf7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir/,STOPPED} 2024-11-13T05:39:28,836 WARN [BP-1057966739-172.17.0.3-1731476313957 heartbeating to localhost/127.0.0.1:40757 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:39:28,836 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:39:28,836 WARN [BP-1057966739-172.17.0.3-1731476313957 heartbeating to localhost/127.0.0.1:40757 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1057966739-172.17.0.3-1731476313957 (Datanode Uuid a46da1c1-de96-42ef-934b-c2e13f618226) service to localhost/127.0.0.1:40757 2024-11-13T05:39:28,836 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:39:28,837 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data3/current/BP-1057966739-172.17.0.3-1731476313957 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:39:28,838 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data4/current/BP-1057966739-172.17.0.3-1731476313957 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:39:28,838 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:39:28,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@166de790{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:39:28,867 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26ab0093{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:39:28,867 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:39:28,868 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@724218ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:39:28,868 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@80048ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir/,STOPPED} 2024-11-13T05:39:28,870 WARN [BP-1057966739-172.17.0.3-1731476313957 heartbeating to localhost/127.0.0.1:40757 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:39:28,870 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:39:28,870 WARN [BP-1057966739-172.17.0.3-1731476313957 heartbeating to localhost/127.0.0.1:40757 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1057966739-172.17.0.3-1731476313957 (Datanode Uuid 5e2c7fb6-cedc-4c11-969b-49535578e07e) service to localhost/127.0.0.1:40757 2024-11-13T05:39:28,870 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:39:28,871 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data1/current/BP-1057966739-172.17.0.3-1731476313957 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:39:28,871 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/cluster_eae190a5-5cbc-0179-435e-7708103e22c4/data/data2/current/BP-1057966739-172.17.0.3-1731476313957 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:39:28,872 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:39:28,880 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54b46454{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:39:28,881 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c2a0d6c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:39:28,881 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:39:28,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48e6714f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:39:28,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@399a62ef{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir/,STOPPED} 2024-11-13T05:39:28,888 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:39:28,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:39:28,913 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=205 (was 179) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:40757 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:40757 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40757 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40757 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40757 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:40757 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a61c20bd729c:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40757 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:40757 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=4665 (was 3105) - AvailableMemoryMB LEAK? - 2024-11-13T05:39:28,919 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=4665 2024-11-13T05:39:28,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.log.dir so I do NOT create it in target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da8ae0b-a249-945e-9f1a-13e18577cf07/hadoop.tmp.dir so I do NOT create it in target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c, deleteOnExit=true 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/test.cache.data in system properties and HBase conf 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:39:28,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:39:28,920 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:39:28,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:39:28,933 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:39:28,973 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:28,973 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:29,049 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:39:29,264 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:29,267 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:39:29,268 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:39:29,268 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:39:29,268 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:39:29,269 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:29,269 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@623cbb81{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:39:29,269 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1cb72b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:39:29,361 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@98b6596{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/java.io.tmpdir/jetty-localhost-32863-hadoop-hdfs-3_4_1-tests_jar-_-any-2780782478832600270/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:39:29,361 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a63bfa6{HTTP/1.1, (http/1.1)}{localhost:32863} 2024-11-13T05:39:29,362 INFO [Time-limited test {}] server.Server(415): Started @251329ms 2024-11-13T05:39:29,372 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:39:29,673 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:29,676 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:39:29,677 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:39:29,677 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:39:29,677 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:39:29,678 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9bd0638{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:39:29,678 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bdfb606{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:39:29,770 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@33cca936{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/java.io.tmpdir/jetty-localhost-42895-hadoop-hdfs-3_4_1-tests_jar-_-any-9368159072451866882/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:39:29,770 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f9d87e6{HTTP/1.1, (http/1.1)}{localhost:42895} 2024-11-13T05:39:29,770 INFO [Time-limited test {}] server.Server(415): Started @251738ms 2024-11-13T05:39:29,771 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:39:29,796 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:29,799 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:39:29,799 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:39:29,799 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:39:29,799 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:39:29,800 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@686a5b12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:39:29,800 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@adc7967{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:39:29,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@41fbd5aa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/java.io.tmpdir/jetty-localhost-34809-hadoop-hdfs-3_4_1-tests_jar-_-any-17378605231712766811/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:39:29,895 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2480e20a{HTTP/1.1, (http/1.1)}{localhost:34809} 2024-11-13T05:39:29,895 INFO [Time-limited test {}] server.Server(415): Started @251863ms 2024-11-13T05:39:29,896 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:39:29,973 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:29,973 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:30,862 WARN [Thread-1970 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data1/current/BP-300909309-172.17.0.3-1731476368937/current, will proceed with Du for space computation calculation, 2024-11-13T05:39:30,862 WARN [Thread-1971 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data2/current/BP-300909309-172.17.0.3-1731476368937/current, will proceed with Du for space computation calculation, 2024-11-13T05:39:30,882 WARN [Thread-1934 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:39:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x22242cf9b87c2571 with lease ID 0xc7c497cf4ebbedd5: Processing first storage report for DS-39aebab3-b621-4d4c-a154-c44337de5fea from datanode DatanodeRegistration(127.0.0.1:41287, datanodeUuid=516f1ee5-5475-4f9f-94b1-3a91ac34d3e5, infoPort=43957, infoSecurePort=0, ipcPort=35937, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937) 2024-11-13T05:39:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x22242cf9b87c2571 with lease ID 0xc7c497cf4ebbedd5: from storage DS-39aebab3-b621-4d4c-a154-c44337de5fea node DatanodeRegistration(127.0.0.1:41287, datanodeUuid=516f1ee5-5475-4f9f-94b1-3a91ac34d3e5, infoPort=43957, infoSecurePort=0, ipcPort=35937, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:39:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x22242cf9b87c2571 with lease ID 0xc7c497cf4ebbedd5: Processing first storage report for DS-4eb1f036-73ff-479e-be9b-b1acd3895d97 from datanode DatanodeRegistration(127.0.0.1:41287, datanodeUuid=516f1ee5-5475-4f9f-94b1-3a91ac34d3e5, infoPort=43957, infoSecurePort=0, ipcPort=35937, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937) 2024-11-13T05:39:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x22242cf9b87c2571 with lease ID 0xc7c497cf4ebbedd5: from storage DS-4eb1f036-73ff-479e-be9b-b1acd3895d97 node DatanodeRegistration(127.0.0.1:41287, datanodeUuid=516f1ee5-5475-4f9f-94b1-3a91ac34d3e5, infoPort=43957, infoSecurePort=0, ipcPort=35937, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:39:30,974 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:30,974 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:30,979 WARN [Thread-1981 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data3/current/BP-300909309-172.17.0.3-1731476368937/current, will proceed with Du for space computation calculation, 2024-11-13T05:39:30,979 WARN [Thread-1982 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data4/current/BP-300909309-172.17.0.3-1731476368937/current, will proceed with Du for space computation calculation, 2024-11-13T05:39:30,996 WARN [Thread-1957 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:39:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80e4b1194e298b8 with lease ID 0xc7c497cf4ebbedd6: Processing first storage report for DS-dbb45dcf-9d2e-4ee7-85e3-70e619143f82 from datanode DatanodeRegistration(127.0.0.1:38147, datanodeUuid=d2a0943d-ef6d-4d6f-b749-165bcca97e46, infoPort=46149, infoSecurePort=0, ipcPort=45875, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937) 2024-11-13T05:39:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80e4b1194e298b8 with lease ID 0xc7c497cf4ebbedd6: from storage DS-dbb45dcf-9d2e-4ee7-85e3-70e619143f82 node DatanodeRegistration(127.0.0.1:38147, datanodeUuid=d2a0943d-ef6d-4d6f-b749-165bcca97e46, infoPort=46149, infoSecurePort=0, ipcPort=45875, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:39:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80e4b1194e298b8 with lease ID 0xc7c497cf4ebbedd6: Processing first storage report for DS-d725dc57-e61c-4376-b4b6-7d1491c8c512 from datanode DatanodeRegistration(127.0.0.1:38147, datanodeUuid=d2a0943d-ef6d-4d6f-b749-165bcca97e46, infoPort=46149, infoSecurePort=0, ipcPort=45875, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937) 2024-11-13T05:39:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80e4b1194e298b8 with lease ID 0xc7c497cf4ebbedd6: from storage DS-d725dc57-e61c-4376-b4b6-7d1491c8c512 node DatanodeRegistration(127.0.0.1:38147, datanodeUuid=d2a0943d-ef6d-4d6f-b749-165bcca97e46, infoPort=46149, infoSecurePort=0, ipcPort=45875, storageInfo=lv=-57;cid=testClusterID;nsid=1167795108;c=1731476368937), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:39:31,028 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416 2024-11-13T05:39:31,032 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/zookeeper_0, clientPort=49847, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:39:31,033 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49847 2024-11-13T05:39:31,034 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,036 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:39:31,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:39:31,047 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3 with version=8 2024-11-13T05:39:31,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:39:31,049 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:39:31,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:39:31,051 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35097 2024-11-13T05:39:31,052 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35097 connecting to ZooKeeper ensemble=127.0.0.1:49847 2024-11-13T05:39:31,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:350970x0, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:39:31,132 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35097-0x10132b4b2020000 connected 2024-11-13T05:39:31,215 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,216 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:39:31,218 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3, hbase.cluster.distributed=false 2024-11-13T05:39:31,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:39:31,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35097 2024-11-13T05:39:31,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35097 2024-11-13T05:39:31,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35097 2024-11-13T05:39:31,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35097 2024-11-13T05:39:31,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35097 2024-11-13T05:39:31,235 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:39:31,235 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:39:31,236 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37093 2024-11-13T05:39:31,237 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37093 connecting to ZooKeeper ensemble=127.0.0.1:49847 2024-11-13T05:39:31,238 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,239 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370930x0, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:39:31,252 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:39:31,252 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37093-0x10132b4b2020001 connected 2024-11-13T05:39:31,252 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:39:31,252 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:39:31,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:39:31,254 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:39:31,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37093 2024-11-13T05:39:31,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37093 2024-11-13T05:39:31,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37093 2024-11-13T05:39:31,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37093 2024-11-13T05:39:31,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37093 2024-11-13T05:39:31,269 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:35097 2024-11-13T05:39:31,269 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:31,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:31,278 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:39:31,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,288 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:39:31,289 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,35097,1731476371049 from backup master directory 2024-11-13T05:39:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:31,298 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:39:31,298 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,303 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/hbase.id] with ID: f1420d1a-572c-452d-ae6b-58d29a014c87 2024-11-13T05:39:31,303 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/.tmp/hbase.id 2024-11-13T05:39:31,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:39:31,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:39:31,311 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/.tmp/hbase.id]:[hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/hbase.id] 2024-11-13T05:39:31,321 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:31,321 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:39:31,322 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-13T05:39:31,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:39:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:39:31,336 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:39:31,336 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:39:31,337 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:39:31,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:39:31,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:39:31,347 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store 2024-11-13T05:39:31,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:39:31,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:39:31,354 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:31,354 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:39:31,354 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:31,354 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:31,354 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:39:31,354 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:31,354 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:31,354 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476371354Disabling compacts and flushes for region at 1731476371354Disabling writes for close at 1731476371354Writing region close event to WAL at 1731476371354Closed at 1731476371354 2024-11-13T05:39:31,355 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/.initializing 2024-11-13T05:39:31,355 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/WALs/a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,357 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C35097%2C1731476371049, suffix=, logDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/WALs/a61c20bd729c,35097,1731476371049, archiveDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/oldWALs, maxLogs=10 2024-11-13T05:39:31,357 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C35097%2C1731476371049.1731476371357 2024-11-13T05:39:31,361 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/WALs/a61c20bd729c,35097,1731476371049/a61c20bd729c%2C35097%2C1731476371049.1731476371357 2024-11-13T05:39:31,364 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43957:43957),(127.0.0.1/127.0.0.1:46149:46149)] 2024-11-13T05:39:31,367 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:39:31,367 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:31,367 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,368 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,369 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:39:31,370 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:31,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:39:31,372 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:31,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:39:31,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:31,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:39:31,375 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:31,376 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,376 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,377 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,378 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,378 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,378 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:39:31,379 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:31,381 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:39:31,381 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809505, jitterRate=0.029339715838432312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:39:31,382 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476371368Initializing all the Stores at 1731476371369 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476371369Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476371369Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476371369Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476371369Cleaning up temporary data from old regions at 1731476371378 (+9 ms)Region opened successfully at 1731476371382 (+4 ms) 2024-11-13T05:39:31,383 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:39:31,387 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d425fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:39:31,387 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:39:31,388 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:39:31,388 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:39:31,388 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:39:31,388 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:39:31,388 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:39:31,388 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:39:31,390 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:39:31,391 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:39:31,404 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:39:31,404 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:39:31,405 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:39:31,414 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:39:31,414 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:39:31,415 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:39:31,425 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:39:31,426 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:39:31,435 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:39:31,437 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:39:31,446 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:39:31,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:39:31,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:39:31,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,457 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,35097,1731476371049, sessionid=0x10132b4b2020000, setting cluster-up flag (Was=false) 2024-11-13T05:39:31,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,509 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:39:31,512 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:31,567 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:39:31,570 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,35097,1731476371049 2024-11-13T05:39:31,573 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:39:31,576 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:31,576 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:39:31,577 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:39:31,577 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,35097,1731476371049 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:39:31,580 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:31,580 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:31,580 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:31,580 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:31,580 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:39:31,581 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,581 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:39:31,581 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476401582 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:39:31,582 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,583 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:31,583 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:39:31,583 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:39:31,583 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:39:31,583 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:39:31,583 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:39:31,584 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:39:31,584 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476371584,5,FailOnTimeoutGroup] 2024-11-13T05:39:31,584 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,584 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476371584,5,FailOnTimeoutGroup] 2024-11-13T05:39:31,584 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,584 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:39:31,584 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,584 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,584 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:39:31,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:39:31,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:39:31,591 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:39:31,591 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3 2024-11-13T05:39:31,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:39:31,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:39:31,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:31,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:39:31,601 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:39:31,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:31,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:39:31,603 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:39:31,603 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:31,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:39:31,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:39:31,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:31,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:39:31,606 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:39:31,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:31,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:31,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:39:31,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740 2024-11-13T05:39:31,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740 2024-11-13T05:39:31,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:39:31,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:39:31,609 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:39:31,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:39:31,612 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:39:31,613 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781161, jitterRate=-0.006703168153762817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:39:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476371598Initializing all the Stores at 1731476371599 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476371599Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476371599Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476371599Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476371599Cleaning up temporary data from old regions at 1731476371609 (+10 ms)Region opened successfully at 1731476371613 (+4 ms) 2024-11-13T05:39:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:39:31,613 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:39:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:39:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:39:31,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:39:31,614 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:39:31,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476371613Disabling compacts and flushes for region at 1731476371613Disabling writes for close at 1731476371613Writing region close event to WAL at 1731476371614 (+1 ms)Closed at 1731476371614 2024-11-13T05:39:31,615 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:31,615 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:39:31,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:39:31,617 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:39:31,618 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:39:31,660 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(746): ClusterId : f1420d1a-572c-452d-ae6b-58d29a014c87 2024-11-13T05:39:31,660 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:39:31,675 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:39:31,675 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:39:31,691 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:39:31,692 DEBUG [RS:0;a61c20bd729c:37093 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8a0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:39:31,711 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:37093 2024-11-13T05:39:31,711 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:39:31,711 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:39:31,711 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:39:31,712 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,35097,1731476371049 with port=37093, startcode=1731476371235 2024-11-13T05:39:31,712 DEBUG [RS:0;a61c20bd729c:37093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:39:31,714 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59557, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:39:31,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35097 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,37093,1731476371235 2024-11-13T05:39:31,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35097 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:31,716 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3 2024-11-13T05:39:31,716 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44363 2024-11-13T05:39:31,716 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:39:31,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:39:31,725 DEBUG [RS:0;a61c20bd729c:37093 {}] zookeeper.ZKUtil(111): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,37093,1731476371235 2024-11-13T05:39:31,725 WARN [RS:0;a61c20bd729c:37093 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:39:31,725 INFO [RS:0;a61c20bd729c:37093 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:39:31,726 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235 2024-11-13T05:39:31,726 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,37093,1731476371235] 2024-11-13T05:39:31,729 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:39:31,731 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:39:31,731 INFO [RS:0;a61c20bd729c:37093 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:39:31,732 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,732 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:39:31,733 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:39:31,733 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,733 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:31,734 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:39:31,734 DEBUG [RS:0;a61c20bd729c:37093 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:39:31,736 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,736 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,736 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,736 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,737 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,737 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,37093,1731476371235-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:39:31,750 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:39:31,750 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,37093,1731476371235-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,750 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,750 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.Replication(171): a61c20bd729c,37093,1731476371235 started 2024-11-13T05:39:31,762 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:31,762 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,37093,1731476371235, RpcServer on a61c20bd729c/172.17.0.3:37093, sessionid=0x10132b4b2020001 2024-11-13T05:39:31,762 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:39:31,762 DEBUG [RS:0;a61c20bd729c:37093 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,37093,1731476371235 2024-11-13T05:39:31,762 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,37093,1731476371235' 2024-11-13T05:39:31,762 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,37093,1731476371235 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,37093,1731476371235' 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:39:31,763 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:39:31,764 DEBUG [RS:0;a61c20bd729c:37093 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:39:31,764 INFO [RS:0;a61c20bd729c:37093 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:39:31,764 INFO [RS:0;a61c20bd729c:37093 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:39:31,769 WARN [a61c20bd729c:35097 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:39:31,868 INFO [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C37093%2C1731476371235, suffix=, logDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235, archiveDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/oldWALs, maxLogs=32 2024-11-13T05:39:31,869 INFO [RS:0;a61c20bd729c:37093 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37093%2C1731476371235.1731476371869 2024-11-13T05:39:31,880 INFO [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476371869 2024-11-13T05:39:31,881 DEBUG [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46149:46149),(127.0.0.1/127.0.0.1:43957:43957)] 2024-11-13T05:39:31,976 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:31,976 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:32,019 DEBUG [a61c20bd729c:35097 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:39:32,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:32,024 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,37093,1731476371235, state=OPENING 2024-11-13T05:39:32,072 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:39:32,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:32,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:32,101 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:39:32,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:32,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:32,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,37093,1731476371235}] 2024-11-13T05:39:32,257 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:39:32,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32963, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:39:32,269 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:39:32,269 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:39:32,270 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C37093%2C1731476371235.meta, suffix=.meta, logDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235, archiveDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/oldWALs, maxLogs=32 2024-11-13T05:39:32,271 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37093%2C1731476371235.meta.1731476372271.meta 2024-11-13T05:39:32,275 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.meta.1731476372271.meta 2024-11-13T05:39:32,276 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43957:43957),(127.0.0.1/127.0.0.1:46149:46149)] 2024-11-13T05:39:32,277 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:39:32,277 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:39:32,277 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:39:32,277 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:39:32,278 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:39:32,278 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:32,278 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:39:32,278 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:39:32,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:39:32,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:39:32,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:32,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:32,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:39:32,281 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:39:32,281 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:32,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:32,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:39:32,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:39:32,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:32,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:32,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:39:32,283 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:39:32,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:32,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:32,284 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:39:32,285 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740 2024-11-13T05:39:32,286 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740 2024-11-13T05:39:32,287 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:39:32,287 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:39:32,288 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:39:32,289 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:39:32,289 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776722, jitterRate=-0.012348160147666931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:39:32,289 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:39:32,290 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476372278Writing region info on filesystem at 1731476372278Initializing all the Stores at 1731476372279 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476372279Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476372279Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476372279Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476372279Cleaning up temporary data from old regions at 1731476372287 (+8 ms)Running coprocessor post-open hooks at 1731476372289 (+2 ms)Region opened successfully at 1731476372290 (+1 ms) 2024-11-13T05:39:32,291 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476372257 2024-11-13T05:39:32,293 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:39:32,293 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:39:32,293 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:32,294 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,37093,1731476371235, state=OPEN 2024-11-13T05:39:32,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:39:32,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:39:32,504 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:32,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:32,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:32,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:39:32,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,37093,1731476371235 in 403 msec 2024-11-13T05:39:32,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:39:32,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 891 msec 2024-11-13T05:39:32,510 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:32,510 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:39:32,511 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:32,511 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,37093,1731476371235, seqNum=-1] 2024-11-13T05:39:32,511 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:32,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37415, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:32,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 942 msec 2024-11-13T05:39:32,517 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476372517, completionTime=-1 2024-11-13T05:39:32,517 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:39:32,517 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476432519 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476492519 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35097,1731476371049-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35097,1731476371049-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,519 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35097,1731476371049-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,520 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:35097, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,520 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,520 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,521 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.224sec 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35097,1731476371049-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:39:32,523 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35097,1731476371049-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:39:32,526 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:39:32,526 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:39:32,526 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,35097,1731476371049-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:32,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d89a4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:32,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,35097,-1 for getting cluster id 2024-11-13T05:39:32,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:32,561 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1420d1a-572c-452d-ae6b-58d29a014c87' 2024-11-13T05:39:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1420d1a-572c-452d-ae6b-58d29a014c87" 2024-11-13T05:39:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141d33bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,35097,-1] 2024-11-13T05:39:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:32,563 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:32,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ffe610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:32,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,37093,1731476371235, seqNum=-1] 2024-11-13T05:39:32,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:32,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:32,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,35097,1731476371049 2024-11-13T05:39:32,569 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:32,572 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:39:32,573 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:39:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a61c20bd729c,35097,1731476371049 2024-11-13T05:39:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b484343 2024-11-13T05:39:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:39:32,576 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:39:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-13T05:39:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-13T05:39:32,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:39:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-13T05:39:32,581 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:39:32,581 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:32,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-13T05:39:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:32,583 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:39:32,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741835_1011 (size=381) 2024-11-13T05:39:32,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741835_1011 (size=381) 2024-11-13T05:39:32,591 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e5129b487fc66c9d923a6d1dd6a6e2c1, NAME => 'TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3 2024-11-13T05:39:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741836_1012 (size=64) 2024-11-13T05:39:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741836_1012 (size=64) 2024-11-13T05:39:32,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:32,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing e5129b487fc66c9d923a6d1dd6a6e2c1, disabling compactions & flushes 2024-11-13T05:39:32,597 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. after waiting 0 ms 2024-11-13T05:39:32,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,597 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e5129b487fc66c9d923a6d1dd6a6e2c1: Waiting for close lock at 1731476372597Disabling compacts and flushes for region at 1731476372597Disabling writes for close at 1731476372597Writing region close event to WAL at 1731476372597Closed at 1731476372597 2024-11-13T05:39:32,599 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:39:32,599 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731476372599"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476372599"}]},"ts":"1731476372599"} 2024-11-13T05:39:32,601 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:39:32,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:39:32,602 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476372602"}]},"ts":"1731476372602"} 2024-11-13T05:39:32,604 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-13T05:39:32,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, ASSIGN}] 2024-11-13T05:39:32,605 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, ASSIGN 2024-11-13T05:39:32,606 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, ASSIGN; state=OFFLINE, location=a61c20bd729c,37093,1731476371235; forceNewPlan=false, retain=false 2024-11-13T05:39:32,758 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e5129b487fc66c9d923a6d1dd6a6e2c1, regionState=OPENING, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:32,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, ASSIGN because future has completed 2024-11-13T05:39:32,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5129b487fc66c9d923a6d1dd6a6e2c1, server=a61c20bd729c,37093,1731476371235}] 2024-11-13T05:39:32,924 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,924 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e5129b487fc66c9d923a6d1dd6a6e2c1, NAME => 'TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:39:32,924 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,925 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:32,925 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,925 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,926 INFO [StoreOpener-e5129b487fc66c9d923a6d1dd6a6e2c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,927 INFO [StoreOpener-e5129b487fc66c9d923a6d1dd6a6e2c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5129b487fc66c9d923a6d1dd6a6e2c1 columnFamilyName info 2024-11-13T05:39:32,928 DEBUG [StoreOpener-e5129b487fc66c9d923a6d1dd6a6e2c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:32,928 INFO [StoreOpener-e5129b487fc66c9d923a6d1dd6a6e2c1-1 {}] regionserver.HStore(327): Store=e5129b487fc66c9d923a6d1dd6a6e2c1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:32,928 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,929 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,930 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,931 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,931 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,933 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,936 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:39:32,936 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e5129b487fc66c9d923a6d1dd6a6e2c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=816946, jitterRate=0.0388016551733017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:39:32,937 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:32,937 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e5129b487fc66c9d923a6d1dd6a6e2c1: Running coprocessor pre-open hook at 1731476372925Writing region info on filesystem at 1731476372925Initializing all the Stores at 1731476372926 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476372926Cleaning up temporary data from old regions at 1731476372931 (+5 ms)Running coprocessor post-open hooks at 1731476372937 (+6 ms)Region opened successfully at 1731476372937 2024-11-13T05:39:32,939 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., pid=6, masterSystemTime=1731476372920 2024-11-13T05:39:32,941 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,941 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:32,942 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e5129b487fc66c9d923a6d1dd6a6e2c1, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:32,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5129b487fc66c9d923a6d1dd6a6e2c1, server=a61c20bd729c,37093,1731476371235 because future has completed 2024-11-13T05:39:32,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:39:32,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e5129b487fc66c9d923a6d1dd6a6e2c1, server=a61c20bd729c,37093,1731476371235 in 181 msec 2024-11-13T05:39:32,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:39:32,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, ASSIGN in 343 msec 2024-11-13T05:39:32,950 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:39:32,950 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476372950"}]},"ts":"1731476372950"} 2024-11-13T05:39:32,952 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-13T05:39:32,953 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:39:32,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 376 msec 2024-11-13T05:39:32,977 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:32,977 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:33,285 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,285 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,286 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,287 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,287 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,288 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,312 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,313 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,313 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,314 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,820 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:39:33,823 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,823 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,824 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,825 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,826 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,827 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,829 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,829 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,849 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,849 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,850 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,850 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,850 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,850 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,857 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:33,977 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:33,977 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:34,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-13T05:39:34,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-13T05:39:34,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-13T05:39:34,978 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:34,978 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:35,979 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:35,979 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:36,981 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:36,981 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:37,730 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:39:37,731 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-13T05:39:37,982 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:37,982 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:38,983 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:38,983 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:39,984 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:39,984 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:40,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:39:40,480 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,480 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,481 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,481 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,482 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,482 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,483 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,484 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,507 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,508 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,508 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,511 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,511 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,512 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,514 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:40,985 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:40,985 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:41,987 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:41,987 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35097 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:42,626 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-13T05:39:42,626 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-13T05:39:42,631 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-13T05:39:42,631 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., hostname=a61c20bd729c,37093,1731476371235, seqNum=2] 2024-11-13T05:39:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:42,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:39:42,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/934b0e1bfffa4c71a19a2d270f7931b3 is 1080, key is row0001/info:/1731476382637/Put/seqid=0 2024-11-13T05:39:42,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741837_1013 (size=12509) 2024-11-13T05:39:42,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741837_1013 (size=12509) 2024-11-13T05:39:42,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/934b0e1bfffa4c71a19a2d270f7931b3 2024-11-13T05:39:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/934b0e1bfffa4c71a19a2d270f7931b3 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/934b0e1bfffa4c71a19a2d270f7931b3 2024-11-13T05:39:42,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/934b0e1bfffa4c71a19a2d270f7931b3, entries=7, sequenceid=11, filesize=12.2 K 2024-11-13T05:39:42,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=19.96 KB/20444 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 41ms, sequenceid=11, compaction requested=false 2024-11-13T05:39:42,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:42,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB 2024-11-13T05:39:42,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/cea2e794be1242709377d5ca2723c989 is 1080, key is row0008/info:/1731476382683/Put/seqid=0 2024-11-13T05:39:42,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741838_1014 (size=26530) 2024-11-13T05:39:42,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741838_1014 (size=26530) 2024-11-13T05:39:42,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/cea2e794be1242709377d5ca2723c989 2024-11-13T05:39:42,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/cea2e794be1242709377d5ca2723c989 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989 2024-11-13T05:39:42,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989, entries=20, sequenceid=34, filesize=25.9 K 2024-11-13T05:39:42,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=5.25 KB/5380 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 19ms, sequenceid=34, compaction requested=false 2024-11-13T05:39:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.1 K, sizeToCheck=16.0 K 2024-11-13T05:39:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989 because midkey is the same as first or last row 2024-11-13T05:39:42,988 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:42,988 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:43,990 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:43,990 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:44,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:39:44,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/9ae8c129da31476cb6bd3071de0eb250 is 1080, key is row0028/info:/1731476382725/Put/seqid=0 2024-11-13T05:39:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741839_1015 (size=12509) 2024-11-13T05:39:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741839_1015 (size=12509) 2024-11-13T05:39:44,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/9ae8c129da31476cb6bd3071de0eb250 2024-11-13T05:39:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/9ae8c129da31476cb6bd3071de0eb250 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/9ae8c129da31476cb6bd3071de0eb250 2024-11-13T05:39:44,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/9ae8c129da31476cb6bd3071de0eb250, entries=7, sequenceid=44, filesize=12.2 K 2024-11-13T05:39:44,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 27ms, sequenceid=44, compaction requested=true 2024-11-13T05:39:44,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.3 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989 because midkey is the same as first or last row 2024-11-13T05:39:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5129b487fc66c9d923a6d1dd6a6e2c1:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:44,773 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:44,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:39:44,774 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 51548 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:44,775 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): e5129b487fc66c9d923a6d1dd6a6e2c1/info is initiating minor compaction (all files) 2024-11-13T05:39:44,775 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5129b487fc66c9d923a6d1dd6a6e2c1/info in TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:44,775 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/934b0e1bfffa4c71a19a2d270f7931b3, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/9ae8c129da31476cb6bd3071de0eb250] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp, totalSize=50.3 K 2024-11-13T05:39:44,775 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 934b0e1bfffa4c71a19a2d270f7931b3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731476382637 2024-11-13T05:39:44,776 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea2e794be1242709377d5ca2723c989, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731476382683 2024-11-13T05:39:44,776 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae8c129da31476cb6bd3071de0eb250, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731476382725 2024-11-13T05:39:44,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/12cab98c20e24bad8cde71b67cb88bea is 1080, key is row0035/info:/1731476384747/Put/seqid=0 2024-11-13T05:39:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741840_1016 (size=17894) 2024-11-13T05:39:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741840_1016 (size=17894) 2024-11-13T05:39:44,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/12cab98c20e24bad8cde71b67cb88bea 2024-11-13T05:39:44,790 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5129b487fc66c9d923a6d1dd6a6e2c1#info#compaction#59 average throughput is 17.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:44,791 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/10f9d383649d48199d3f7718b7f2ba85 is 1080, key is row0001/info:/1731476382637/Put/seqid=0 2024-11-13T05:39:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/12cab98c20e24bad8cde71b67cb88bea as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/12cab98c20e24bad8cde71b67cb88bea 2024-11-13T05:39:44,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741841_1017 (size=41747) 2024-11-13T05:39:44,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741841_1017 (size=41747) 2024-11-13T05:39:44,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/12cab98c20e24bad8cde71b67cb88bea, entries=12, sequenceid=59, filesize=17.5 K 2024-11-13T05:39:44,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 26ms, sequenceid=59, compaction requested=false 2024-11-13T05:39:44,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:44,800 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.8 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,800 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,800 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989 because midkey is the same as first or last row 2024-11-13T05:39:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:44,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:39:44,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/ebd85c85803f456abf736ce158ae73e8 is 1080, key is row0047/info:/1731476384776/Put/seqid=0 2024-11-13T05:39:44,807 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/10f9d383649d48199d3f7718b7f2ba85 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 2024-11-13T05:39:44,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741842_1018 (size=17894) 2024-11-13T05:39:44,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741842_1018 (size=17894) 2024-11-13T05:39:44,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/ebd85c85803f456abf736ce158ae73e8 2024-11-13T05:39:44,819 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5129b487fc66c9d923a6d1dd6a6e2c1/info of e5129b487fc66c9d923a6d1dd6a6e2c1 into 10f9d383649d48199d3f7718b7f2ba85(size=40.8 K), total size for store is 58.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:44,819 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:44,819 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., storeName=e5129b487fc66c9d923a6d1dd6a6e2c1/info, priority=13, startTime=1731476384773; duration=0sec 2024-11-13T05:39:44,819 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=58.2 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,819 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,819 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 because midkey is the same as first or last row 2024-11-13T05:39:44,819 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=58.2 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 because midkey is the same as first or last row 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=58.2 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 because midkey is the same as first or last row 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:44,820 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5129b487fc66c9d923a6d1dd6a6e2c1:info 2024-11-13T05:39:44,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/ebd85c85803f456abf736ce158ae73e8 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/ebd85c85803f456abf736ce158ae73e8 2024-11-13T05:39:44,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/ebd85c85803f456abf736ce158ae73e8, entries=12, sequenceid=74, filesize=17.5 K 2024-11-13T05:39:44,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=6.30 KB/6456 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 28ms, sequenceid=74, compaction requested=true 2024-11-13T05:39:44,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:44,831 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=75.7 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,831 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,831 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 because midkey is the same as first or last row 2024-11-13T05:39:44,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5129b487fc66c9d923a6d1dd6a6e2c1:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:44,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:44,831 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:44,832 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 77535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:44,832 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): e5129b487fc66c9d923a6d1dd6a6e2c1/info is initiating minor compaction (all files) 2024-11-13T05:39:44,832 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5129b487fc66c9d923a6d1dd6a6e2c1/info in TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:44,832 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/12cab98c20e24bad8cde71b67cb88bea, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/ebd85c85803f456abf736ce158ae73e8] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp, totalSize=75.7 K 2024-11-13T05:39:44,833 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10f9d383649d48199d3f7718b7f2ba85, keycount=34, bloomtype=ROW, size=40.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731476382637 2024-11-13T05:39:44,833 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12cab98c20e24bad8cde71b67cb88bea, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731476384747 2024-11-13T05:39:44,833 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd85c85803f456abf736ce158ae73e8, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731476384776 2024-11-13T05:39:44,844 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5129b487fc66c9d923a6d1dd6a6e2c1#info#compaction#61 average throughput is 29.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:44,844 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/a568c4dbc37642e18d7cd514624db36c is 1080, key is row0001/info:/1731476382637/Put/seqid=0 2024-11-13T05:39:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741843_1019 (size=67766) 2024-11-13T05:39:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741843_1019 (size=67766) 2024-11-13T05:39:44,854 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/a568c4dbc37642e18d7cd514624db36c as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c 2024-11-13T05:39:44,860 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5129b487fc66c9d923a6d1dd6a6e2c1/info of e5129b487fc66c9d923a6d1dd6a6e2c1 into a568c4dbc37642e18d7cd514624db36c(size=66.2 K), total size for store is 66.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:44,860 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:44,860 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., storeName=e5129b487fc66c9d923a6d1dd6a6e2c1/info, priority=13, startTime=1731476384831; duration=0sec 2024-11-13T05:39:44,860 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.2 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,860 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,860 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c because midkey is the same as first or last row 2024-11-13T05:39:44,860 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.2 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c because midkey is the same as first or last row 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.2 K, sizeToCheck=16.0 K 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c because midkey is the same as first or last row 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:44,861 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5129b487fc66c9d923a6d1dd6a6e2c1:info 2024-11-13T05:39:44,991 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:44,991 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:45,992 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:45,992 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:46,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:39:46,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/531f977771ba4e92b9ab5a3d999ce3e7 is 1080, key is row0059/info:/1731476384804/Put/seqid=0 2024-11-13T05:39:46,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741844_1020 (size=12509) 2024-11-13T05:39:46,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741844_1020 (size=12509) 2024-11-13T05:39:46,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/531f977771ba4e92b9ab5a3d999ce3e7 2024-11-13T05:39:46,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/531f977771ba4e92b9ab5a3d999ce3e7 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/531f977771ba4e92b9ab5a3d999ce3e7 2024-11-13T05:39:46,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/531f977771ba4e92b9ab5a3d999ce3e7, entries=7, sequenceid=86, filesize=12.2 K 2024-11-13T05:39:46,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 28ms, sequenceid=86, compaction requested=false 2024-11-13T05:39:46,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:46,854 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K 2024-11-13T05:39:46,854 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:46,854 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c because midkey is the same as first or last row 2024-11-13T05:39:46,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:39:46,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/fa278a568e8240c59a953f630ebcf7b5 is 1080, key is row0066/info:/1731476386829/Put/seqid=0 2024-11-13T05:39:46,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741845_1021 (size=17894) 2024-11-13T05:39:46,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741845_1021 (size=17894) 2024-11-13T05:39:46,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/fa278a568e8240c59a953f630ebcf7b5 2024-11-13T05:39:46,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/fa278a568e8240c59a953f630ebcf7b5 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/fa278a568e8240c59a953f630ebcf7b5 2024-11-13T05:39:46,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/fa278a568e8240c59a953f630ebcf7b5, entries=12, sequenceid=101, filesize=17.5 K 2024-11-13T05:39:46,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 22ms, sequenceid=101, compaction requested=true 2024-11-13T05:39:46,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:46,876 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=95.9 K, sizeToCheck=16.0 K 2024-11-13T05:39:46,876 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c because midkey is the same as first or last row 2024-11-13T05:39:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5129b487fc66c9d923a6d1dd6a6e2c1:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:46,877 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:46,878 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 98169 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:46,878 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): e5129b487fc66c9d923a6d1dd6a6e2c1/info is initiating minor compaction (all files) 2024-11-13T05:39:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:46,878 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5129b487fc66c9d923a6d1dd6a6e2c1/info in TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:46,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:39:46,878 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/531f977771ba4e92b9ab5a3d999ce3e7, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/fa278a568e8240c59a953f630ebcf7b5] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp, totalSize=95.9 K 2024-11-13T05:39:46,879 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting a568c4dbc37642e18d7cd514624db36c, keycount=58, bloomtype=ROW, size=66.2 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731476382637 2024-11-13T05:39:46,879 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531f977771ba4e92b9ab5a3d999ce3e7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731476384804 2024-11-13T05:39:46,879 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa278a568e8240c59a953f630ebcf7b5, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731476386829 2024-11-13T05:39:46,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/8dbd88fbce664ae184fd0497641e2937 is 1080, key is row0078/info:/1731476386855/Put/seqid=0 2024-11-13T05:39:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741846_1022 (size=17894) 2024-11-13T05:39:46,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741846_1022 (size=17894) 2024-11-13T05:39:46,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/8dbd88fbce664ae184fd0497641e2937 2024-11-13T05:39:46,893 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5129b487fc66c9d923a6d1dd6a6e2c1#info#compaction#65 average throughput is 26.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:46,894 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/2f6d3b43b48d4ef7805528247832e88d is 1080, key is row0001/info:/1731476382637/Put/seqid=0 2024-11-13T05:39:46,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/8dbd88fbce664ae184fd0497641e2937 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/8dbd88fbce664ae184fd0497641e2937 2024-11-13T05:39:46,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741847_1023 (size=88408) 2024-11-13T05:39:46,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741847_1023 (size=88408) 2024-11-13T05:39:46,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/8dbd88fbce664ae184fd0497641e2937, entries=12, sequenceid=116, filesize=17.5 K 2024-11-13T05:39:46,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 23ms, sequenceid=116, compaction requested=false 2024-11-13T05:39:46,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:46,902 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=113.3 K, sizeToCheck=16.0 K 2024-11-13T05:39:46,902 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:46,902 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c because midkey is the same as first or last row 2024-11-13T05:39:46,902 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/2f6d3b43b48d4ef7805528247832e88d as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d 2024-11-13T05:39:46,908 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5129b487fc66c9d923a6d1dd6a6e2c1/info of e5129b487fc66c9d923a6d1dd6a6e2c1 into 2f6d3b43b48d4ef7805528247832e88d(size=86.3 K), total size for store is 103.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5129b487fc66c9d923a6d1dd6a6e2c1: 2024-11-13T05:39:46,908 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., storeName=e5129b487fc66c9d923a6d1dd6a6e2c1/info, priority=13, startTime=1731476386877; duration=0sec 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K 2024-11-13T05:39:46,908 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-13T05:39:46,909 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:46,909 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:46,909 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5129b487fc66c9d923a6d1dd6a6e2c1:info 2024-11-13T05:39:46,932 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35097 {}] assignment.AssignmentManager(1355): Split request from a61c20bd729c,37093,1731476371235, parent={ENCODED => e5129b487fc66c9d923a6d1dd6a6e2c1, NAME => 'TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-13T05:39:46,936 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35097 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:46,939 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35097 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e5129b487fc66c9d923a6d1dd6a6e2c1, daughterA=c7c7b817c66214d9ababce9a0a0b8f71, daughterB=8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:46,940 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e5129b487fc66c9d923a6d1dd6a6e2c1, daughterA=c7c7b817c66214d9ababce9a0a0b8f71, daughterB=8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:46,940 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e5129b487fc66c9d923a6d1dd6a6e2c1, daughterA=c7c7b817c66214d9ababce9a0a0b8f71, daughterB=8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:46,940 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e5129b487fc66c9d923a6d1dd6a6e2c1, daughterA=c7c7b817c66214d9ababce9a0a0b8f71, daughterB=8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:46,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, UNASSIGN}] 2024-11-13T05:39:46,948 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, UNASSIGN 2024-11-13T05:39:46,949 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e5129b487fc66c9d923a6d1dd6a6e2c1, regionState=CLOSING, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:46,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, UNASSIGN because future has completed 2024-11-13T05:39:46,952 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-13T05:39:46,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5129b487fc66c9d923a6d1dd6a6e2c1, server=a61c20bd729c,37093,1731476371235}] 2024-11-13T05:39:46,994 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:46,994 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:47,111 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,111 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-13T05:39:47,112 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing e5129b487fc66c9d923a6d1dd6a6e2c1, disabling compactions & flushes 2024-11-13T05:39:47,112 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:47,112 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:47,112 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. after waiting 0 ms 2024-11-13T05:39:47,112 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:47,112 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing e5129b487fc66c9d923a6d1dd6a6e2c1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:39:47,119 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/71b6a383468f4471b05c20a6a14564c5 is 1080, key is row0090/info:/1731476386879/Put/seqid=0 2024-11-13T05:39:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741848_1024 (size=12509) 2024-11-13T05:39:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741848_1024 (size=12509) 2024-11-13T05:39:47,124 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/71b6a383468f4471b05c20a6a14564c5 2024-11-13T05:39:47,131 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/.tmp/info/71b6a383468f4471b05c20a6a14564c5 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/71b6a383468f4471b05c20a6a14564c5 2024-11-13T05:39:47,138 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/71b6a383468f4471b05c20a6a14564c5, entries=7, sequenceid=127, filesize=12.2 K 2024-11-13T05:39:47,139 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 27ms, sequenceid=127, compaction requested=true 2024-11-13T05:39:47,141 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/934b0e1bfffa4c71a19a2d270f7931b3, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/9ae8c129da31476cb6bd3071de0eb250, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/12cab98c20e24bad8cde71b67cb88bea, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/ebd85c85803f456abf736ce158ae73e8, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/531f977771ba4e92b9ab5a3d999ce3e7, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/fa278a568e8240c59a953f630ebcf7b5] to archive 2024-11-13T05:39:47,142 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:39:47,144 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/934b0e1bfffa4c71a19a2d270f7931b3 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/934b0e1bfffa4c71a19a2d270f7931b3 2024-11-13T05:39:47,146 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/cea2e794be1242709377d5ca2723c989 2024-11-13T05:39:47,148 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/10f9d383649d48199d3f7718b7f2ba85 2024-11-13T05:39:47,150 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/9ae8c129da31476cb6bd3071de0eb250 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/9ae8c129da31476cb6bd3071de0eb250 2024-11-13T05:39:47,152 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/12cab98c20e24bad8cde71b67cb88bea to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/12cab98c20e24bad8cde71b67cb88bea 2024-11-13T05:39:47,154 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/a568c4dbc37642e18d7cd514624db36c 2024-11-13T05:39:47,155 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/ebd85c85803f456abf736ce158ae73e8 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/ebd85c85803f456abf736ce158ae73e8 2024-11-13T05:39:47,157 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/531f977771ba4e92b9ab5a3d999ce3e7 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/531f977771ba4e92b9ab5a3d999ce3e7 2024-11-13T05:39:47,159 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/fa278a568e8240c59a953f630ebcf7b5 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/fa278a568e8240c59a953f630ebcf7b5 2024-11-13T05:39:47,168 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1 2024-11-13T05:39:47,169 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. 2024-11-13T05:39:47,169 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for e5129b487fc66c9d923a6d1dd6a6e2c1: Waiting for close lock at 1731476387112Running coprocessor pre-close hooks at 1731476387112Disabling compacts and flushes for region at 1731476387112Disabling writes for close at 1731476387112Obtaining lock to block concurrent updates at 1731476387112Preparing flush snapshotting stores in e5129b487fc66c9d923a6d1dd6a6e2c1 at 1731476387112Finished memstore snapshotting TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., syncing WAL and waiting on mvcc, flushsize=dataSize=7532, getHeapSize=8304, getOffHeapSize=0, getCellsCount=7 at 1731476387113 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. at 1731476387114 (+1 ms)Flushing e5129b487fc66c9d923a6d1dd6a6e2c1/info: creating writer at 1731476387114Flushing e5129b487fc66c9d923a6d1dd6a6e2c1/info: appending metadata at 1731476387118 (+4 ms)Flushing e5129b487fc66c9d923a6d1dd6a6e2c1/info: closing flushed file at 1731476387118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33088f23: reopening flushed file at 1731476387129 (+11 ms)Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for e5129b487fc66c9d923a6d1dd6a6e2c1 in 27ms, sequenceid=127, compaction requested=true at 1731476387139 (+10 ms)Writing region close event to WAL at 1731476387163 (+24 ms)Running coprocessor post-close hooks at 1731476387169 (+6 ms)Closed at 1731476387169 2024-11-13T05:39:47,172 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e5129b487fc66c9d923a6d1dd6a6e2c1, regionState=CLOSED 2024-11-13T05:39:47,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5129b487fc66c9d923a6d1dd6a6e2c1, server=a61c20bd729c,37093,1731476371235 because future has completed 2024-11-13T05:39:47,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-13T05:39:47,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure e5129b487fc66c9d923a6d1dd6a6e2c1, server=a61c20bd729c,37093,1731476371235 in 225 msec 2024-11-13T05:39:47,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:39:47,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e5129b487fc66c9d923a6d1dd6a6e2c1, UNASSIGN in 235 msec 2024-11-13T05:39:47,197 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:47,203 INFO [PEWorker-3 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=e5129b487fc66c9d923a6d1dd6a6e2c1, threads=3 2024-11-13T05:39:47,206 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d for region: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,206 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/8dbd88fbce664ae184fd0497641e2937 for region: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,206 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/71b6a383468f4471b05c20a6a14564c5 for region: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,215 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/71b6a383468f4471b05c20a6a14564c5, top=true 2024-11-13T05:39:47,215 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/8dbd88fbce664ae184fd0497641e2937, top=true 2024-11-13T05:39:47,227 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937 for child: 8e926ff3e44dfe3a0683e37fb14b10d2, parent: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,227 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5 for child: 8e926ff3e44dfe3a0683e37fb14b10d2, parent: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,227 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/8dbd88fbce664ae184fd0497641e2937 for region: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,227 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/71b6a383468f4471b05c20a6a14564c5 for region: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741849_1025 (size=27) 2024-11-13T05:39:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741849_1025 (size=27) 2024-11-13T05:39:47,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741850_1026 (size=27) 2024-11-13T05:39:47,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741850_1026 (size=27) 2024-11-13T05:39:47,244 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d for region: e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:39:47,246 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region e5129b487fc66c9d923a6d1dd6a6e2c1 Daughter A: [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1] storefiles, Daughter B: [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937] storefiles. 2024-11-13T05:39:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741851_1027 (size=71) 2024-11-13T05:39:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741851_1027 (size=71) 2024-11-13T05:39:47,257 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:47,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741852_1028 (size=71) 2024-11-13T05:39:47,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741852_1028 (size=71) 2024-11-13T05:39:47,271 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:47,281 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1 2024-11-13T05:39:47,284 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1 2024-11-13T05:39:47,286 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731476387285"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731476387285"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731476387285"}]},"ts":"1731476387285"} 2024-11-13T05:39:47,286 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731476387285"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476387285"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731476387285"}]},"ts":"1731476387285"} 2024-11-13T05:39:47,286 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731476387285"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476387285"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731476387285"}]},"ts":"1731476387285"} 2024-11-13T05:39:47,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c7c7b817c66214d9ababce9a0a0b8f71, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8e926ff3e44dfe3a0683e37fb14b10d2, ASSIGN}] 2024-11-13T05:39:47,303 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c7c7b817c66214d9ababce9a0a0b8f71, ASSIGN 2024-11-13T05:39:47,304 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8e926ff3e44dfe3a0683e37fb14b10d2, ASSIGN 2024-11-13T05:39:47,304 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c7c7b817c66214d9ababce9a0a0b8f71, ASSIGN; state=SPLITTING_NEW, location=a61c20bd729c,37093,1731476371235; forceNewPlan=false, retain=false 2024-11-13T05:39:47,304 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8e926ff3e44dfe3a0683e37fb14b10d2, ASSIGN; state=SPLITTING_NEW, location=a61c20bd729c,37093,1731476371235; forceNewPlan=false, retain=false 2024-11-13T05:39:47,455 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8e926ff3e44dfe3a0683e37fb14b10d2, regionState=OPENING, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:47,455 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c7c7b817c66214d9ababce9a0a0b8f71, regionState=OPENING, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:47,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8e926ff3e44dfe3a0683e37fb14b10d2, ASSIGN because future has completed 2024-11-13T05:39:47,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e926ff3e44dfe3a0683e37fb14b10d2, server=a61c20bd729c,37093,1731476371235}] 2024-11-13T05:39:47,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c7c7b817c66214d9ababce9a0a0b8f71, ASSIGN because future has completed 2024-11-13T05:39:47,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7c7b817c66214d9ababce9a0a0b8f71, server=a61c20bd729c,37093,1731476371235}] 2024-11-13T05:39:47,616 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:39:47,617 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => c7c7b817c66214d9ababce9a0a0b8f71, NAME => 'TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.', STARTKEY => '', ENDKEY => 'row0062'} 2024-11-13T05:39:47,617 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,617 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:47,618 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,618 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,620 INFO [StoreOpener-c7c7b817c66214d9ababce9a0a0b8f71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,622 INFO [StoreOpener-c7c7b817c66214d9ababce9a0a0b8f71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7c7b817c66214d9ababce9a0a0b8f71 columnFamilyName info 2024-11-13T05:39:47,622 DEBUG [StoreOpener-c7c7b817c66214d9ababce9a0a0b8f71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:47,639 DEBUG [StoreOpener-c7c7b817c66214d9ababce9a0a0b8f71-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1->hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d-bottom 2024-11-13T05:39:47,640 INFO [StoreOpener-c7c7b817c66214d9ababce9a0a0b8f71-1 {}] regionserver.HStore(327): Store=c7c7b817c66214d9ababce9a0a0b8f71/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:47,640 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,641 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,643 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,643 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,643 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,645 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,645 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened c7c7b817c66214d9ababce9a0a0b8f71; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=726204, jitterRate=-0.0765848308801651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:39:47,646 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:39:47,646 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for c7c7b817c66214d9ababce9a0a0b8f71: Running coprocessor pre-open hook at 1731476387618Writing region info on filesystem at 1731476387618Initializing all the Stores at 1731476387620 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476387620Cleaning up temporary data from old regions at 1731476387643 (+23 ms)Running coprocessor post-open hooks at 1731476387646 (+3 ms)Region opened successfully at 1731476387646 2024-11-13T05:39:47,647 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71., pid=13, masterSystemTime=1731476387610 2024-11-13T05:39:47,647 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store c7c7b817c66214d9ababce9a0a0b8f71:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:47,647 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:47,647 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T05:39:47,648 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:39:47,648 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): c7c7b817c66214d9ababce9a0a0b8f71/info is initiating minor compaction (all files) 2024-11-13T05:39:47,648 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7c7b817c66214d9ababce9a0a0b8f71/info in TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:39:47,648 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1->hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d-bottom] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/.tmp, totalSize=86.3 K 2024-11-13T05:39:47,648 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1, keycount=38, bloomtype=ROW, size=86.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731476382637 2024-11-13T05:39:47,704 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:39:47,705 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:39:47,705 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:39:47,705 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e926ff3e44dfe3a0683e37fb14b10d2, NAME => 'TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.', STARTKEY => 'row0062', ENDKEY => ''} 2024-11-13T05:39:47,706 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,706 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:47,706 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,706 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c7c7b817c66214d9ababce9a0a0b8f71, regionState=OPEN, openSeqNum=131, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:47,709 INFO [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-13T05:39:47,710 INFO [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e926ff3e44dfe3a0683e37fb14b10d2 columnFamilyName info 2024-11-13T05:39:47,710 DEBUG [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:47,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:47,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-11-13T05:39:47,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7c7b817c66214d9ababce9a0a0b8f71, server=a61c20bd729c,37093,1731476371235 because future has completed 2024-11-13T05:39:47,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-11-13T05:39:47,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure c7c7b817c66214d9ababce9a0a0b8f71, server=a61c20bd729c,37093,1731476371235 in 253 msec 2024-11-13T05:39:47,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c7c7b817c66214d9ababce9a0a0b8f71, ASSIGN in 414 msec 2024-11-13T05:39:47,721 DEBUG [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1->hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d-top 2024-11-13T05:39:47,725 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7c7b817c66214d9ababce9a0a0b8f71#info#compaction#67 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:47,726 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/.tmp/info/8185d3b4517643d68efdfe2c4613e758 is 1080, key is row0001/info:/1731476382637/Put/seqid=0 2024-11-13T05:39:47,727 DEBUG [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5 2024-11-13T05:39:47,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/info/e1866cad73cb405883a8bc1a84d8ea0e is 193, key is TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2./info:regioninfo/1731476387455/Put/seqid=0 2024-11-13T05:39:47,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741853_1029 (size=70862) 2024-11-13T05:39:47,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741853_1029 (size=70862) 2024-11-13T05:39:47,732 DEBUG [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937 2024-11-13T05:39:47,732 INFO [StoreOpener-8e926ff3e44dfe3a0683e37fb14b10d2-1 {}] regionserver.HStore(327): Store=8e926ff3e44dfe3a0683e37fb14b10d2/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:47,732 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,733 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,734 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,734 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741854_1030 (size=9847) 2024-11-13T05:39:47,734 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741854_1030 (size=9847) 2024-11-13T05:39:47,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/info/e1866cad73cb405883a8bc1a84d8ea0e 2024-11-13T05:39:47,736 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,737 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 8e926ff3e44dfe3a0683e37fb14b10d2; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844672, jitterRate=0.07405690848827362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:39:47,737 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:47,737 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 8e926ff3e44dfe3a0683e37fb14b10d2: Running coprocessor pre-open hook at 1731476387706Writing region info on filesystem at 1731476387707 (+1 ms)Initializing all the Stores at 1731476387708 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476387708Cleaning up temporary data from old regions at 1731476387734 (+26 ms)Running coprocessor post-open hooks at 1731476387737 (+3 ms)Region opened successfully at 1731476387737 2024-11-13T05:39:47,738 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., pid=12, masterSystemTime=1731476387610 2024-11-13T05:39:47,738 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:47,738 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:47,738 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:47,739 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:39:47,739 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:39:47,739 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:39:47,739 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1->hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d-top, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=116.0 K 2024-11-13T05:39:47,740 DEBUG [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:39:47,740 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1, keycount=38, bloomtype=ROW, size=86.3 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731476382637 2024-11-13T05:39:47,740 INFO [RS_OPEN_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:39:47,740 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731476386855 2024-11-13T05:39:47,741 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731476386879 2024-11-13T05:39:47,741 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8e926ff3e44dfe3a0683e37fb14b10d2, regionState=OPEN, openSeqNum=131, regionLocation=a61c20bd729c,37093,1731476371235 2024-11-13T05:39:47,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e926ff3e44dfe3a0683e37fb14b10d2, server=a61c20bd729c,37093,1731476371235 because future has completed 2024-11-13T05:39:47,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T05:39:47,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e926ff3e44dfe3a0683e37fb14b10d2, server=a61c20bd729c,37093,1731476371235 in 286 msec 2024-11-13T05:39:47,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7 2024-11-13T05:39:47,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8e926ff3e44dfe3a0683e37fb14b10d2, ASSIGN in 444 msec 2024-11-13T05:39:47,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=e5129b487fc66c9d923a6d1dd6a6e2c1, daughterA=c7c7b817c66214d9ababce9a0a0b8f71, daughterB=8e926ff3e44dfe3a0683e37fb14b10d2 in 813 msec 2024-11-13T05:39:47,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/ns/20d000cd44b042d085e17b5313ec0725 is 43, key is default/ns:d/1731476372513/Put/seqid=0 2024-11-13T05:39:47,761 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#70 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:47,762 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/64b931270b484fd19b2e4a6b70259a5f is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:39:47,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741855_1031 (size=5153) 2024-11-13T05:39:47,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741855_1031 (size=5153) 2024-11-13T05:39:47,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/ns/20d000cd44b042d085e17b5313ec0725 2024-11-13T05:39:47,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741856_1032 (size=42984) 2024-11-13T05:39:47,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741856_1032 (size=42984) 2024-11-13T05:39:47,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/table/930f749d962b474889c02773acb846ce is 65, key is TestLogRolling-testLogRolling/table:state/1731476372950/Put/seqid=0 2024-11-13T05:39:47,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741857_1033 (size=5340) 2024-11-13T05:39:47,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741857_1033 (size=5340) 2024-11-13T05:39:47,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/table/930f749d962b474889c02773acb846ce 2024-11-13T05:39:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/info/e1866cad73cb405883a8bc1a84d8ea0e as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/info/e1866cad73cb405883a8bc1a84d8ea0e 2024-11-13T05:39:47,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/info/e1866cad73cb405883a8bc1a84d8ea0e, entries=30, sequenceid=17, filesize=9.6 K 2024-11-13T05:39:47,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/ns/20d000cd44b042d085e17b5313ec0725 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/ns/20d000cd44b042d085e17b5313ec0725 2024-11-13T05:39:47,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/ns/20d000cd44b042d085e17b5313ec0725, entries=2, sequenceid=17, filesize=5.0 K 2024-11-13T05:39:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/table/930f749d962b474889c02773acb846ce as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/table/930f749d962b474889c02773acb846ce 2024-11-13T05:39:47,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/table/930f749d962b474889c02773acb846ce, entries=2, sequenceid=17, filesize=5.2 K 2024-11-13T05:39:47,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 102ms, sequenceid=17, compaction requested=false 2024-11-13T05:39:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-13T05:39:47,994 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:47,994 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:48,146 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/.tmp/info/8185d3b4517643d68efdfe2c4613e758 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/8185d3b4517643d68efdfe2c4613e758 2024-11-13T05:39:48,154 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in c7c7b817c66214d9ababce9a0a0b8f71/info of c7c7b817c66214d9ababce9a0a0b8f71 into 8185d3b4517643d68efdfe2c4613e758(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:48,154 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7c7b817c66214d9ababce9a0a0b8f71: 2024-11-13T05:39:48,154 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71., storeName=c7c7b817c66214d9ababce9a0a0b8f71/info, priority=15, startTime=1731476387647; duration=0sec 2024-11-13T05:39:48,155 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:48,155 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7c7b817c66214d9ababce9a0a0b8f71:info 2024-11-13T05:39:48,175 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/64b931270b484fd19b2e4a6b70259a5f as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/64b931270b484fd19b2e4a6b70259a5f 2024-11-13T05:39:48,181 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into 64b931270b484fd19b2e4a6b70259a5f(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:48,181 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:39:48,181 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476387738; duration=0sec 2024-11-13T05:39:48,181 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:48,181 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:39:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:32932 deadline: 1731476398896, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. is not online on a61c20bd729c,37093,1731476371235 2024-11-13T05:39:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., hostname=a61c20bd729c,37093,1731476371235, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., hostname=a61c20bd729c,37093,1731476371235, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. is not online on a61c20bd729c,37093,1731476371235 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., hostname=a61c20bd729c,37093,1731476371235, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1. is not online on a61c20bd729c,37093,1731476371235 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731476372576.e5129b487fc66c9d923a6d1dd6a6e2c1., hostname=a61c20bd729c,37093,1731476371235, seqNum=2 from cache 2024-11-13T05:39:48,995 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:48,995 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:49,996 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:49,996 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:50,997 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:50,997 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:51,998 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:51,998 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:52,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,171 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,171 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,172 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,175 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,176 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,196 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,200 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,200 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,200 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,202 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T05:39:52,713 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,714 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,715 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,716 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,717 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,717 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,737 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,737 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,737 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,737 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,738 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,738 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,741 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,741 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,741 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,743 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:39:52,999 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:52,999 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:54,001 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:54,001 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:55,002 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:55,002 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:56,003 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:56,003 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:57,004 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:57,004 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:58,005 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:58,005 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:59,006 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:59,006 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:39:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., hostname=a61c20bd729c,37093,1731476371235, seqNum=131] 2024-11-13T05:39:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:59,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:39:59,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/544443c5a6764e279545a8fb225db04d is 1080, key is row0097/info:/1731476399012/Put/seqid=0 2024-11-13T05:39:59,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741858_1034 (size=12516) 2024-11-13T05:39:59,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741858_1034 (size=12516) 2024-11-13T05:39:59,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/544443c5a6764e279545a8fb225db04d 2024-11-13T05:39:59,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/544443c5a6764e279545a8fb225db04d as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/544443c5a6764e279545a8fb225db04d 2024-11-13T05:39:59,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/544443c5a6764e279545a8fb225db04d, entries=7, sequenceid=141, filesize=12.2 K 2024-11-13T05:39:59,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 23ms, sequenceid=141, compaction requested=false 2024-11-13T05:39:59,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:39:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:39:59,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:39:59,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/3d90d54d25d74ac8925917b409f774f4 is 1080, key is row0104/info:/1731476399025/Put/seqid=0 2024-11-13T05:39:59,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741859_1035 (size=17906) 2024-11-13T05:39:59,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741859_1035 (size=17906) 2024-11-13T05:39:59,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/3d90d54d25d74ac8925917b409f774f4 2024-11-13T05:39:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/3d90d54d25d74ac8925917b409f774f4 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3d90d54d25d74ac8925917b409f774f4 2024-11-13T05:39:59,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3d90d54d25d74ac8925917b409f774f4, entries=12, sequenceid=156, filesize=17.5 K 2024-11-13T05:39:59,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=13.66 KB/13988 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 160ms, sequenceid=156, compaction requested=true 2024-11-13T05:39:59,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:39:59,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:59,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:59,209 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:59,210 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 73406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:59,210 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:39:59,210 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:39:59,210 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/64b931270b484fd19b2e4a6b70259a5f, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/544443c5a6764e279545a8fb225db04d, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3d90d54d25d74ac8925917b409f774f4] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=71.7 K 2024-11-13T05:39:59,210 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64b931270b484fd19b2e4a6b70259a5f, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731476384813 2024-11-13T05:39:59,211 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 544443c5a6764e279545a8fb225db04d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476399012 2024-11-13T05:39:59,211 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d90d54d25d74ac8925917b409f774f4, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731476399025 2024-11-13T05:39:59,222 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#74 average throughput is 27.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:59,223 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4e1eb8f6aa6246e18eeb2a552f547ce5 is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:39:59,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741860_1036 (size=63636) 2024-11-13T05:39:59,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741860_1036 (size=63636) 2024-11-13T05:39:59,233 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4e1eb8f6aa6246e18eeb2a552f547ce5 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4e1eb8f6aa6246e18eeb2a552f547ce5 2024-11-13T05:39:59,238 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into 4e1eb8f6aa6246e18eeb2a552f547ce5(size=62.1 K), total size for store is 62.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:59,238 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:39:59,238 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476399209; duration=0sec 2024-11-13T05:39:59,238 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:59,238 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:00,007 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:00,007 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:01,009 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:01,009 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:01,028 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:40:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:01,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-13T05:40:01,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/f3835158c147441cbf413a4948f59342 is 1080, key is row0116/info:/1731476399049/Put/seqid=0 2024-11-13T05:40:01,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741861_1037 (size=20078) 2024-11-13T05:40:01,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741861_1037 (size=20078) 2024-11-13T05:40:01,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/f3835158c147441cbf413a4948f59342 2024-11-13T05:40:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/f3835158c147441cbf413a4948f59342 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f3835158c147441cbf413a4948f59342 2024-11-13T05:40:01,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f3835158c147441cbf413a4948f59342, entries=14, sequenceid=174, filesize=19.6 K 2024-11-13T05:40:01,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 25ms, sequenceid=174, compaction requested=false 2024-11-13T05:40:01,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:01,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:40:01,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/ebdf18a389fe4325a1e447a3b1fc38a0 is 1080, key is row0130/info:/1731476401080/Put/seqid=0 2024-11-13T05:40:01,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741862_1038 (size=17906) 2024-11-13T05:40:01,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741862_1038 (size=17906) 2024-11-13T05:40:01,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/ebdf18a389fe4325a1e447a3b1fc38a0 2024-11-13T05:40:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/ebdf18a389fe4325a1e447a3b1fc38a0 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ebdf18a389fe4325a1e447a3b1fc38a0 2024-11-13T05:40:01,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ebdf18a389fe4325a1e447a3b1fc38a0, entries=12, sequenceid=189, filesize=17.5 K 2024-11-13T05:40:01,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=8.41 KB/8608 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 37ms, sequenceid=189, compaction requested=true 2024-11-13T05:40:01,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:01,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:01,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:01,141 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:01,142 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101620 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:01,142 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:40:01,142 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:01,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-11-13T05:40:01,142 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4e1eb8f6aa6246e18eeb2a552f547ce5, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f3835158c147441cbf413a4948f59342, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ebdf18a389fe4325a1e447a3b1fc38a0] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=99.2 K 2024-11-13T05:40:01,142 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1eb8f6aa6246e18eeb2a552f547ce5, keycount=54, bloomtype=ROW, size=62.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731476384813 2024-11-13T05:40:01,143 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.Compactor(225): Compacting f3835158c147441cbf413a4948f59342, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731476399049 2024-11-13T05:40:01,143 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdf18a389fe4325a1e447a3b1fc38a0, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731476401080 2024-11-13T05:40:01,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/3060440e348b46018d37b745b6b6634e is 1080, key is row0142/info:/1731476401104/Put/seqid=0 2024-11-13T05:40:01,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741863_1039 (size=14672) 2024-11-13T05:40:01,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741863_1039 (size=14672) 2024-11-13T05:40:01,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/3060440e348b46018d37b745b6b6634e 2024-11-13T05:40:01,177 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#78 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:01,178 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4da4eed3e4c94232a6d8d255952688da is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:40:01,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/3060440e348b46018d37b745b6b6634e as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3060440e348b46018d37b745b6b6634e 2024-11-13T05:40:01,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3060440e348b46018d37b745b6b6634e, entries=9, sequenceid=201, filesize=14.3 K 2024-11-13T05:40:01,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=10.51 KB/10760 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 42ms, sequenceid=201, compaction requested=false 2024-11-13T05:40:01,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:01,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741864_1040 (size=91843) 2024-11-13T05:40:01,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741864_1040 (size=91843) 2024-11-13T05:40:01,597 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4da4eed3e4c94232a6d8d255952688da as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4da4eed3e4c94232a6d8d255952688da 2024-11-13T05:40:01,602 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into 4da4eed3e4c94232a6d8d255952688da(size=89.7 K), total size for store is 104.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:01,602 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:01,602 INFO [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476401141; duration=0sec 2024-11-13T05:40:01,602 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:01,602 DEBUG [RS:0;a61c20bd729c:37093-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:02,010 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:02,010 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:03,011 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:03,011 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:03,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-13T05:40:03,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/733615d2315c4ebeb8020a466ea0e74b is 1080, key is row0151/info:/1731476401143/Put/seqid=0 2024-11-13T05:40:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741865_1041 (size=16828) 2024-11-13T05:40:03,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741865_1041 (size=16828) 2024-11-13T05:40:03,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/733615d2315c4ebeb8020a466ea0e74b 2024-11-13T05:40:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/733615d2315c4ebeb8020a466ea0e74b as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/733615d2315c4ebeb8020a466ea0e74b 2024-11-13T05:40:03,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/733615d2315c4ebeb8020a466ea0e74b, entries=11, sequenceid=216, filesize=16.4 K 2024-11-13T05:40:03,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 25ms, sequenceid=216, compaction requested=true 2024-11-13T05:40:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,198 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:03,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-13T05:40:03,199 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 123343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:03,199 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:40:03,199 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:03,199 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4da4eed3e4c94232a6d8d255952688da, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3060440e348b46018d37b745b6b6634e, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/733615d2315c4ebeb8020a466ea0e74b] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=120.5 K 2024-11-13T05:40:03,200 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4da4eed3e4c94232a6d8d255952688da, keycount=80, bloomtype=ROW, size=89.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731476384813 2024-11-13T05:40:03,200 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3060440e348b46018d37b745b6b6634e, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731476401104 2024-11-13T05:40:03,200 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733615d2315c4ebeb8020a466ea0e74b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731476401143 2024-11-13T05:40:03,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/1ea357ed410b4bf99644afdd0001d16f is 1080, key is row0162/info:/1731476403174/Put/seqid=0 2024-11-13T05:40:03,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741866_1042 (size=16828) 2024-11-13T05:40:03,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741866_1042 (size=16828) 2024-11-13T05:40:03,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/1ea357ed410b4bf99644afdd0001d16f 2024-11-13T05:40:03,213 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#81 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:03,213 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/65746751be67443ea445b3f7691a9833 is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:40:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/1ea357ed410b4bf99644afdd0001d16f as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/1ea357ed410b4bf99644afdd0001d16f 2024-11-13T05:40:03,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/1ea357ed410b4bf99644afdd0001d16f, entries=11, sequenceid=230, filesize=16.4 K 2024-11-13T05:40:03,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 24ms, sequenceid=230, compaction requested=false 2024-11-13T05:40:03,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:03,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-13T05:40:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741867_1043 (size=113509) 2024-11-13T05:40:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741867_1043 (size=113509) 2024-11-13T05:40:03,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4ee00d4fd461468aba932081393ef9bc is 1080, key is row0173/info:/1731476403199/Put/seqid=0 2024-11-13T05:40:03,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741868_1044 (size=19000) 2024-11-13T05:40:03,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4ee00d4fd461468aba932081393ef9bc 2024-11-13T05:40:03,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741868_1044 (size=19000) 2024-11-13T05:40:03,234 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/65746751be67443ea445b3f7691a9833 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/65746751be67443ea445b3f7691a9833 2024-11-13T05:40:03,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4ee00d4fd461468aba932081393ef9bc as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ee00d4fd461468aba932081393ef9bc 2024-11-13T05:40:03,240 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into 65746751be67443ea445b3f7691a9833(size=110.8 K), total size for store is 127.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:03,240 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:03,240 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476403197; duration=0sec 2024-11-13T05:40:03,240 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,240 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:03,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ee00d4fd461468aba932081393ef9bc, entries=13, sequenceid=246, filesize=18.6 K 2024-11-13T05:40:03,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=7.36 KB/7532 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 20ms, sequenceid=246, compaction requested=true 2024-11-13T05:40:03,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:03,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:03,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,244 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:03,245 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 149337 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:03,245 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:40:03,245 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:03,245 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/65746751be67443ea445b3f7691a9833, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/1ea357ed410b4bf99644afdd0001d16f, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ee00d4fd461468aba932081393ef9bc] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=145.8 K 2024-11-13T05:40:03,245 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65746751be67443ea445b3f7691a9833, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731476384813 2024-11-13T05:40:03,246 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea357ed410b4bf99644afdd0001d16f, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731476403174 2024-11-13T05:40:03,246 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee00d4fd461468aba932081393ef9bc, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731476403199 2024-11-13T05:40:03,256 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#83 average throughput is 63.62 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:03,257 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4ecb60a68e954f0fa366d4fb93eec58e is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:40:03,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741869_1045 (size=139688) 2024-11-13T05:40:03,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741869_1045 (size=139688) 2024-11-13T05:40:03,266 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/4ecb60a68e954f0fa366d4fb93eec58e as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ecb60a68e954f0fa366d4fb93eec58e 2024-11-13T05:40:03,272 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into 4ecb60a68e954f0fa366d4fb93eec58e(size=136.4 K), total size for store is 136.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:03,272 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:03,272 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476403244; duration=0sec 2024-11-13T05:40:03,272 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,272 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:04,012 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:04,012 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:05,013 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:05,013 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:05,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-11-13T05:40:05,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/650c18a3e6184065b25bbe4a7cb933af is 1080, key is row0186/info:/1731476403224/Put/seqid=0 2024-11-13T05:40:05,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741870_1046 (size=13595) 2024-11-13T05:40:05,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741870_1046 (size=13595) 2024-11-13T05:40:05,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/650c18a3e6184065b25bbe4a7cb933af 2024-11-13T05:40:05,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/650c18a3e6184065b25bbe4a7cb933af as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/650c18a3e6184065b25bbe4a7cb933af 2024-11-13T05:40:05,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/650c18a3e6184065b25bbe4a7cb933af, entries=8, sequenceid=259, filesize=13.3 K 2024-11-13T05:40:05,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 25ms, sequenceid=259, compaction requested=false 2024-11-13T05:40:05,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:05,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-13T05:40:05,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/489aec82b2494e68ab0da4b2f173b91a is 1080, key is row0194/info:/1731476405247/Put/seqid=0 2024-11-13T05:40:05,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741871_1047 (size=17918) 2024-11-13T05:40:05,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741871_1047 (size=17918) 2024-11-13T05:40:05,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/489aec82b2494e68ab0da4b2f173b91a 2024-11-13T05:40:05,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/489aec82b2494e68ab0da4b2f173b91a as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/489aec82b2494e68ab0da4b2f173b91a 2024-11-13T05:40:05,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/489aec82b2494e68ab0da4b2f173b91a, entries=12, sequenceid=274, filesize=17.5 K 2024-11-13T05:40:05,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 21ms, sequenceid=274, compaction requested=true 2024-11-13T05:40:05,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:05,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:05,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:05,291 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:05,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-13T05:40:05,292 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 171201 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:05,292 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:40:05,293 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:05,293 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ecb60a68e954f0fa366d4fb93eec58e, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/650c18a3e6184065b25bbe4a7cb933af, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/489aec82b2494e68ab0da4b2f173b91a] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=167.2 K 2024-11-13T05:40:05,293 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ecb60a68e954f0fa366d4fb93eec58e, keycount=124, bloomtype=ROW, size=136.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731476384813 2024-11-13T05:40:05,293 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 650c18a3e6184065b25bbe4a7cb933af, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731476403224 2024-11-13T05:40:05,294 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489aec82b2494e68ab0da4b2f173b91a, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731476405247 2024-11-13T05:40:05,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/f16d3b99e443493c9dcbfb96670b41e6 is 1080, key is row0206/info:/1731476405271/Put/seqid=0 2024-11-13T05:40:05,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741872_1048 (size=16839) 2024-11-13T05:40:05,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741872_1048 (size=16839) 2024-11-13T05:40:05,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/f16d3b99e443493c9dcbfb96670b41e6 2024-11-13T05:40:05,307 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#87 average throughput is 36.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:05,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/f16d3b99e443493c9dcbfb96670b41e6 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f16d3b99e443493c9dcbfb96670b41e6 2024-11-13T05:40:05,307 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/a41436dba53f4dc2857e10fca9b1684b is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:40:05,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f16d3b99e443493c9dcbfb96670b41e6, entries=11, sequenceid=288, filesize=16.4 K 2024-11-13T05:40:05,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 22ms, sequenceid=288, compaction requested=false 2024-11-13T05:40:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:05,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741873_1049 (size=161367) 2024-11-13T05:40:05,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741873_1049 (size=161367) 2024-11-13T05:40:05,325 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/a41436dba53f4dc2857e10fca9b1684b as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/a41436dba53f4dc2857e10fca9b1684b 2024-11-13T05:40:05,331 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into a41436dba53f4dc2857e10fca9b1684b(size=157.6 K), total size for store is 174.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:05,332 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:05,332 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476405291; duration=0sec 2024-11-13T05:40:05,332 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:05,332 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:06,014 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:06,014 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:07,015 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:07,015 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:07,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-11-13T05:40:07,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/744c69f547c24ce7a8eae1c0ad16f9b6 is 1080, key is row0217/info:/1731476405292/Put/seqid=0 2024-11-13T05:40:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741874_1050 (size=14681) 2024-11-13T05:40:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741874_1050 (size=14681) 2024-11-13T05:40:07,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/744c69f547c24ce7a8eae1c0ad16f9b6 2024-11-13T05:40:07,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/744c69f547c24ce7a8eae1c0ad16f9b6 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/744c69f547c24ce7a8eae1c0ad16f9b6 2024-11-13T05:40:07,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/744c69f547c24ce7a8eae1c0ad16f9b6, entries=9, sequenceid=301, filesize=14.3 K 2024-11-13T05:40:07,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=12.61 KB/12912 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 26ms, sequenceid=301, compaction requested=true 2024-11-13T05:40:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:07,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:07,341 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:07,342 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 192887 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:07,342 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:40:07,342 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:07,343 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/a41436dba53f4dc2857e10fca9b1684b, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f16d3b99e443493c9dcbfb96670b41e6, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/744c69f547c24ce7a8eae1c0ad16f9b6] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=188.4 K 2024-11-13T05:40:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:07,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-13T05:40:07,343 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting a41436dba53f4dc2857e10fca9b1684b, keycount=144, bloomtype=ROW, size=157.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731476384813 2024-11-13T05:40:07,343 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16d3b99e443493c9dcbfb96670b41e6, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731476405271 2024-11-13T05:40:07,343 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744c69f547c24ce7a8eae1c0ad16f9b6, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731476405292 2024-11-13T05:40:07,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/d5135191e8304664890a622b0ea4850b is 1080, key is row0226/info:/1731476407318/Put/seqid=0 2024-11-13T05:40:07,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741875_1051 (size=19013) 2024-11-13T05:40:07,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741875_1051 (size=19013) 2024-11-13T05:40:07,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/d5135191e8304664890a622b0ea4850b 2024-11-13T05:40:07,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/d5135191e8304664890a622b0ea4850b as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/d5135191e8304664890a622b0ea4850b 2024-11-13T05:40:07,358 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#90 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:07,359 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/ee9c3a0b62a947b7bd0e7379433d09ea is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:40:07,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/d5135191e8304664890a622b0ea4850b, entries=13, sequenceid=317, filesize=18.6 K 2024-11-13T05:40:07,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=9.46 KB/9684 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 20ms, sequenceid=317, compaction requested=false 2024-11-13T05:40:07,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:07,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741876_1052 (size=183053) 2024-11-13T05:40:07,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741876_1052 (size=183053) 2024-11-13T05:40:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37093 {}] regionserver.HRegion(8855): Flush requested on 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:07,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-13T05:40:07,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/329796b03aea43af82af01d7dffa7e8f is 1080, key is row0239/info:/1731476407344/Put/seqid=0 2024-11-13T05:40:07,372 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/ee9c3a0b62a947b7bd0e7379433d09ea as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ee9c3a0b62a947b7bd0e7379433d09ea 2024-11-13T05:40:07,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741877_1053 (size=16839) 2024-11-13T05:40:07,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741877_1053 (size=16839) 2024-11-13T05:40:07,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/329796b03aea43af82af01d7dffa7e8f 2024-11-13T05:40:07,379 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into ee9c3a0b62a947b7bd0e7379433d09ea(size=178.8 K), total size for store is 197.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:07,379 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:07,380 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476407341; duration=0sec 2024-11-13T05:40:07,380 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:07,380 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/329796b03aea43af82af01d7dffa7e8f as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/329796b03aea43af82af01d7dffa7e8f 2024-11-13T05:40:07,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/329796b03aea43af82af01d7dffa7e8f, entries=11, sequenceid=331, filesize=16.4 K 2024-11-13T05:40:07,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=7.36 KB/7532 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 21ms, sequenceid=331, compaction requested=true 2024-11-13T05:40:07,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:07,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e926ff3e44dfe3a0683e37fb14b10d2:info, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:07,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:07,387 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:07,388 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 218905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:07,388 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1541): 8e926ff3e44dfe3a0683e37fb14b10d2/info is initiating minor compaction (all files) 2024-11-13T05:40:07,388 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e926ff3e44dfe3a0683e37fb14b10d2/info in TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:07,388 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ee9c3a0b62a947b7bd0e7379433d09ea, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/d5135191e8304664890a622b0ea4850b, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/329796b03aea43af82af01d7dffa7e8f] into tmpdir=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp, totalSize=213.8 K 2024-11-13T05:40:07,388 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9c3a0b62a947b7bd0e7379433d09ea, keycount=164, bloomtype=ROW, size=178.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731476384813 2024-11-13T05:40:07,389 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5135191e8304664890a622b0ea4850b, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731476407318 2024-11-13T05:40:07,389 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] compactions.Compactor(225): Compacting 329796b03aea43af82af01d7dffa7e8f, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731476407344 2024-11-13T05:40:07,400 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e926ff3e44dfe3a0683e37fb14b10d2#info#compaction#92 average throughput is 38.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:07,401 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/e6a940cbaf0f41449877e86f2e527bc8 is 1080, key is row0062/info:/1731476384813/Put/seqid=0 2024-11-13T05:40:07,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741878_1054 (size=209128) 2024-11-13T05:40:07,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741878_1054 (size=209128) 2024-11-13T05:40:07,408 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/e6a940cbaf0f41449877e86f2e527bc8 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/e6a940cbaf0f41449877e86f2e527bc8 2024-11-13T05:40:07,414 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e926ff3e44dfe3a0683e37fb14b10d2/info of 8e926ff3e44dfe3a0683e37fb14b10d2 into e6a940cbaf0f41449877e86f2e527bc8(size=204.2 K), total size for store is 204.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:07,414 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:07,414 INFO [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., storeName=8e926ff3e44dfe3a0683e37fb14b10d2/info, priority=13, startTime=1731476407387; duration=0sec 2024-11-13T05:40:07,414 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:07,414 DEBUG [RS:0;a61c20bd729c:37093-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e926ff3e44dfe3a0683e37fb14b10d2:info 2024-11-13T05:40:08,015 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:08,015 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:09,017 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:09,017 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:09,382 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-13T05:40:09,383 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37093%2C1731476371235.1731476409383 2024-11-13T05:40:09,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,446 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476371869 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476409383 2024-11-13T05:40:09,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43957:43957),(127.0.0.1/127.0.0.1:46149:46149)] 2024-11-13T05:40:09,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476371869 is not closed yet, will try archiving it next time 2024-11-13T05:40:09,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741833_1009 (size=317837) 2024-11-13T05:40:09,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741833_1009 (size=317837) 2024-11-13T05:40:09,483 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e926ff3e44dfe3a0683e37fb14b10d2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-13T05:40:09,489 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/30980ffb554e4be89732255234a896b9 is 1080, key is row0250/info:/1731476407367/Put/seqid=0 2024-11-13T05:40:09,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741880_1056 (size=12523) 2024-11-13T05:40:09,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741880_1056 (size=12523) 2024-11-13T05:40:09,497 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/30980ffb554e4be89732255234a896b9 2024-11-13T05:40:09,505 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/.tmp/info/30980ffb554e4be89732255234a896b9 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/30980ffb554e4be89732255234a896b9 2024-11-13T05:40:09,510 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/30980ffb554e4be89732255234a896b9, entries=7, sequenceid=343, filesize=12.2 K 2024-11-13T05:40:09,511 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 8e926ff3e44dfe3a0683e37fb14b10d2 in 28ms, sequenceid=343, compaction requested=false 2024-11-13T05:40:09,512 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e926ff3e44dfe3a0683e37fb14b10d2: 2024-11-13T05:40:09,512 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-11-13T05:40:09,515 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/info/ecafeb8419f044a9ada32968c893f24e is 193, key is TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2./info:regioninfo/1731476387741/Put/seqid=0 2024-11-13T05:40:09,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741881_1057 (size=6223) 2024-11-13T05:40:09,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741881_1057 (size=6223) 2024-11-13T05:40:09,520 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/info/ecafeb8419f044a9ada32968c893f24e 2024-11-13T05:40:09,524 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/.tmp/info/ecafeb8419f044a9ada32968c893f24e as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/info/ecafeb8419f044a9ada32968c893f24e 2024-11-13T05:40:09,528 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/info/ecafeb8419f044a9ada32968c893f24e, entries=5, sequenceid=21, filesize=6.1 K 2024-11-13T05:40:09,529 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 17ms, sequenceid=21, compaction requested=false 2024-11-13T05:40:09,529 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-13T05:40:09,529 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c7c7b817c66214d9ababce9a0a0b8f71: 2024-11-13T05:40:09,530 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C37093%2C1731476371235.1731476409530 2024-11-13T05:40:09,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,535 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476409383 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476409530 2024-11-13T05:40:09,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741879_1055 (size=731) 2024-11-13T05:40:09,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741879_1055 (size=731) 2024-11-13T05:40:09,540 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476371869 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/oldWALs/a61c20bd729c%2C37093%2C1731476371235.1731476371869 2024-11-13T05:40:09,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43957:43957),(127.0.0.1/127.0.0.1:46149:46149)] 2024-11-13T05:40:09,541 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:40:09,541 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/WALs/a61c20bd729c,37093,1731476371235/a61c20bd729c%2C37093%2C1731476371235.1731476409383 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/oldWALs/a61c20bd729c%2C37093%2C1731476371235.1731476409383 2024-11-13T05:40:09,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:40:09,542 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:40:09,542 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:09,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:09,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:09,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:09,542 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:40:09,542 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=605387780, stopped=false 2024-11-13T05:40:09,542 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,35097,1731476371049 2024-11-13T05:40:09,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:40:09,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:40:09,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:09,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:09,580 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:40:09,580 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:40:09,580 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:09,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:09,580 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,37093,1731476371235' ***** 2024-11-13T05:40:09,581 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:40:09,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:40:09,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:40:09,581 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e926ff3e44dfe3a0683e37fb14b10d2 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(3091): Received CLOSE for c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,37093,1731476371235 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:40:09,581 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e926ff3e44dfe3a0683e37fb14b10d2, disabling compactions & flushes 2024-11-13T05:40:09,581 INFO [RS:0;a61c20bd729c:37093 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:37093. 2024-11-13T05:40:09,582 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:09,582 DEBUG [RS:0;a61c20bd729c:37093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. after waiting 0 ms 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:09,582 DEBUG [RS:0;a61c20bd729c:37093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:09,582 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:40:09,582 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:40:09,582 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:40:09,582 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:40:09,582 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-13T05:40:09,582 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1325): Online Regions={8e926ff3e44dfe3a0683e37fb14b10d2=TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2., 1588230740=hbase:meta,,1.1588230740, c7c7b817c66214d9ababce9a0a0b8f71=TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.} 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:40:09,582 DEBUG [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8e926ff3e44dfe3a0683e37fb14b10d2, c7c7b817c66214d9ababce9a0a0b8f71 2024-11-13T05:40:09,582 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:40:09,582 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:40:09,582 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1->hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d-top, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/64b931270b484fd19b2e4a6b70259a5f, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/544443c5a6764e279545a8fb225db04d, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4e1eb8f6aa6246e18eeb2a552f547ce5, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3d90d54d25d74ac8925917b409f774f4, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f3835158c147441cbf413a4948f59342, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4da4eed3e4c94232a6d8d255952688da, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ebdf18a389fe4325a1e447a3b1fc38a0, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3060440e348b46018d37b745b6b6634e, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/65746751be67443ea445b3f7691a9833, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/733615d2315c4ebeb8020a466ea0e74b, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/1ea357ed410b4bf99644afdd0001d16f, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ecb60a68e954f0fa366d4fb93eec58e, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ee00d4fd461468aba932081393ef9bc, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/650c18a3e6184065b25bbe4a7cb933af, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/a41436dba53f4dc2857e10fca9b1684b, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/489aec82b2494e68ab0da4b2f173b91a, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f16d3b99e443493c9dcbfb96670b41e6, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ee9c3a0b62a947b7bd0e7379433d09ea, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/744c69f547c24ce7a8eae1c0ad16f9b6, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/d5135191e8304664890a622b0ea4850b, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/329796b03aea43af82af01d7dffa7e8f] to archive 2024-11-13T05:40:09,583 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:09,585 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:40:09,586 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-13T05:40:09,586 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-8dbd88fbce664ae184fd0497641e2937 2024-11-13T05:40:09,586 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:40:09,586 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:40:09,586 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476409582Running coprocessor pre-close hooks at 1731476409582Disabling compacts and flushes for region at 1731476409582Disabling writes for close at 1731476409582Writing region close event to WAL at 1731476409583 (+1 ms)Running coprocessor post-close hooks at 1731476409586 (+3 ms)Closed at 1731476409586 2024-11-13T05:40:09,587 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:40:09,587 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/64b931270b484fd19b2e4a6b70259a5f to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/64b931270b484fd19b2e4a6b70259a5f 2024-11-13T05:40:09,588 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/TestLogRolling-testLogRolling=e5129b487fc66c9d923a6d1dd6a6e2c1-71b6a383468f4471b05c20a6a14564c5 2024-11-13T05:40:09,589 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/544443c5a6764e279545a8fb225db04d to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/544443c5a6764e279545a8fb225db04d 2024-11-13T05:40:09,591 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4e1eb8f6aa6246e18eeb2a552f547ce5 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4e1eb8f6aa6246e18eeb2a552f547ce5 2024-11-13T05:40:09,592 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3d90d54d25d74ac8925917b409f774f4 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3d90d54d25d74ac8925917b409f774f4 2024-11-13T05:40:09,593 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f3835158c147441cbf413a4948f59342 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f3835158c147441cbf413a4948f59342 2024-11-13T05:40:09,594 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4da4eed3e4c94232a6d8d255952688da to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4da4eed3e4c94232a6d8d255952688da 2024-11-13T05:40:09,595 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ebdf18a389fe4325a1e447a3b1fc38a0 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ebdf18a389fe4325a1e447a3b1fc38a0 2024-11-13T05:40:09,596 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3060440e348b46018d37b745b6b6634e to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/3060440e348b46018d37b745b6b6634e 2024-11-13T05:40:09,597 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/65746751be67443ea445b3f7691a9833 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/65746751be67443ea445b3f7691a9833 2024-11-13T05:40:09,598 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/733615d2315c4ebeb8020a466ea0e74b to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/733615d2315c4ebeb8020a466ea0e74b 2024-11-13T05:40:09,599 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/1ea357ed410b4bf99644afdd0001d16f to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/1ea357ed410b4bf99644afdd0001d16f 2024-11-13T05:40:09,600 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ecb60a68e954f0fa366d4fb93eec58e to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ecb60a68e954f0fa366d4fb93eec58e 2024-11-13T05:40:09,601 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ee00d4fd461468aba932081393ef9bc to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/4ee00d4fd461468aba932081393ef9bc 2024-11-13T05:40:09,602 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/650c18a3e6184065b25bbe4a7cb933af to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/650c18a3e6184065b25bbe4a7cb933af 2024-11-13T05:40:09,603 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/a41436dba53f4dc2857e10fca9b1684b to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/a41436dba53f4dc2857e10fca9b1684b 2024-11-13T05:40:09,604 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/489aec82b2494e68ab0da4b2f173b91a to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/489aec82b2494e68ab0da4b2f173b91a 2024-11-13T05:40:09,605 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f16d3b99e443493c9dcbfb96670b41e6 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/f16d3b99e443493c9dcbfb96670b41e6 2024-11-13T05:40:09,606 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ee9c3a0b62a947b7bd0e7379433d09ea to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/ee9c3a0b62a947b7bd0e7379433d09ea 2024-11-13T05:40:09,607 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/744c69f547c24ce7a8eae1c0ad16f9b6 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/744c69f547c24ce7a8eae1c0ad16f9b6 2024-11-13T05:40:09,608 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/d5135191e8304664890a622b0ea4850b to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/d5135191e8304664890a622b0ea4850b 2024-11-13T05:40:09,609 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/329796b03aea43af82af01d7dffa7e8f to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/info/329796b03aea43af82af01d7dffa7e8f 2024-11-13T05:40:09,609 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a61c20bd729c:35097 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-13T05:40:09,610 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [64b931270b484fd19b2e4a6b70259a5f=42984, 544443c5a6764e279545a8fb225db04d=12516, 4e1eb8f6aa6246e18eeb2a552f547ce5=63636, 3d90d54d25d74ac8925917b409f774f4=17906, f3835158c147441cbf413a4948f59342=20078, 4da4eed3e4c94232a6d8d255952688da=91843, ebdf18a389fe4325a1e447a3b1fc38a0=17906, 3060440e348b46018d37b745b6b6634e=14672, 65746751be67443ea445b3f7691a9833=113509, 733615d2315c4ebeb8020a466ea0e74b=16828, 1ea357ed410b4bf99644afdd0001d16f=16828, 4ecb60a68e954f0fa366d4fb93eec58e=139688, 4ee00d4fd461468aba932081393ef9bc=19000, 650c18a3e6184065b25bbe4a7cb933af=13595, a41436dba53f4dc2857e10fca9b1684b=161367, 489aec82b2494e68ab0da4b2f173b91a=17918, f16d3b99e443493c9dcbfb96670b41e6=16839, ee9c3a0b62a947b7bd0e7379433d09ea=183053, 744c69f547c24ce7a8eae1c0ad16f9b6=14681, d5135191e8304664890a622b0ea4850b=19013, 329796b03aea43af82af01d7dffa7e8f=16839] 2024-11-13T05:40:09,613 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/8e926ff3e44dfe3a0683e37fb14b10d2/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130 2024-11-13T05:40:09,613 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:09,613 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e926ff3e44dfe3a0683e37fb14b10d2: Waiting for close lock at 1731476409581Running coprocessor pre-close hooks at 1731476409581Disabling compacts and flushes for region at 1731476409581Disabling writes for close at 1731476409582 (+1 ms)Writing region close event to WAL at 1731476409610 (+28 ms)Running coprocessor post-close hooks at 1731476409613 (+3 ms)Closed at 1731476409613 2024-11-13T05:40:09,614 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731476386936.8e926ff3e44dfe3a0683e37fb14b10d2. 2024-11-13T05:40:09,614 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c7c7b817c66214d9ababce9a0a0b8f71, disabling compactions & flushes 2024-11-13T05:40:09,614 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:40:09,614 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:40:09,614 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. after waiting 0 ms 2024-11-13T05:40:09,614 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:40:09,614 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1->hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/e5129b487fc66c9d923a6d1dd6a6e2c1/info/2f6d3b43b48d4ef7805528247832e88d-bottom] to archive 2024-11-13T05:40:09,615 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:09,616 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1 to hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/archive/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/info/2f6d3b43b48d4ef7805528247832e88d.e5129b487fc66c9d923a6d1dd6a6e2c1 2024-11-13T05:40:09,616 WARN [StoreCloser-TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-13T05:40:09,620 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/data/default/TestLogRolling-testLogRolling/c7c7b817c66214d9ababce9a0a0b8f71/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130 2024-11-13T05:40:09,621 INFO [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:40:09,621 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c7c7b817c66214d9ababce9a0a0b8f71: Waiting for close lock at 1731476409614Running coprocessor pre-close hooks at 1731476409614Disabling compacts and flushes for region at 1731476409614Disabling writes for close at 1731476409614Writing region close event to WAL at 1731476409617 (+3 ms)Running coprocessor post-close hooks at 1731476409621 (+4 ms)Closed at 1731476409621 2024-11-13T05:40:09,621 DEBUG [RS_CLOSE_REGION-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731476386936.c7c7b817c66214d9ababce9a0a0b8f71. 2024-11-13T05:40:09,739 INFO [regionserver/a61c20bd729c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:40:09,782 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,37093,1731476371235; all regions closed. 2024-11-13T05:40:09,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,785 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:40:09,785 INFO [regionserver/a61c20bd729c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:40:09,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741834_1010 (size=8107) 2024-11-13T05:40:09,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741834_1010 (size=8107) 2024-11-13T05:40:09,792 DEBUG [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/oldWALs 2024-11-13T05:40:09,792 INFO [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C37093%2C1731476371235.meta:.meta(num 1731476372271) 2024-11-13T05:40:09,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741882_1058 (size=780) 2024-11-13T05:40:09,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741882_1058 (size=780) 2024-11-13T05:40:09,798 DEBUG [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/oldWALs 2024-11-13T05:40:09,798 INFO [RS:0;a61c20bd729c:37093 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C37093%2C1731476371235:(num 1731476409530) 2024-11-13T05:40:09,798 DEBUG [RS:0;a61c20bd729c:37093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:09,798 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:40:09,798 INFO [RS:0;a61c20bd729c:37093 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:40:09,799 INFO [RS:0;a61c20bd729c:37093 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:40:09,799 INFO [RS:0;a61c20bd729c:37093 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:40:09,799 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:40:09,799 INFO [RS:0;a61c20bd729c:37093 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37093 2024-11-13T05:40:09,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,37093,1731476371235 2024-11-13T05:40:09,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:40:09,811 INFO [RS:0;a61c20bd729c:37093 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:40:09,822 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,37093,1731476371235] 2024-11-13T05:40:09,832 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,37093,1731476371235 already deleted, retry=false 2024-11-13T05:40:09,832 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,37093,1731476371235 expired; onlineServers=0 2024-11-13T05:40:09,832 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,35097,1731476371049' ***** 2024-11-13T05:40:09,832 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:40:09,833 INFO [M:0;a61c20bd729c:35097 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:40:09,833 INFO [M:0;a61c20bd729c:35097 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:40:09,833 DEBUG [M:0;a61c20bd729c:35097 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:40:09,833 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:40:09,833 DEBUG [M:0;a61c20bd729c:35097 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:40:09,833 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476371584 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476371584,5,FailOnTimeoutGroup] 2024-11-13T05:40:09,833 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476371584 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476371584,5,FailOnTimeoutGroup] 2024-11-13T05:40:09,833 INFO [M:0;a61c20bd729c:35097 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:40:09,833 INFO [M:0;a61c20bd729c:35097 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:40:09,834 DEBUG [M:0;a61c20bd729c:35097 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:40:09,834 INFO [M:0;a61c20bd729c:35097 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:40:09,834 INFO [M:0;a61c20bd729c:35097 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:40:09,834 INFO [M:0;a61c20bd729c:35097 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:40:09,835 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:40:09,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:40:09,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:09,843 DEBUG [M:0;a61c20bd729c:35097 {}] zookeeper.ZKUtil(347): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:40:09,843 WARN [M:0;a61c20bd729c:35097 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:40:09,844 INFO [M:0;a61c20bd729c:35097 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/.lastflushedseqids 2024-11-13T05:40:09,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741883_1059 (size=228) 2024-11-13T05:40:09,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741883_1059 (size=228) 2024-11-13T05:40:09,854 INFO [M:0;a61c20bd729c:35097 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:40:09,854 INFO [M:0;a61c20bd729c:35097 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:40:09,854 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:40:09,854 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:09,854 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:09,854 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:40:09,854 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:09,855 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB 2024-11-13T05:40:09,869 DEBUG [M:0;a61c20bd729c:35097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/56ce54216b3b41e0bafa1ffc58f0b6c1 is 82, key is hbase:meta,,1/info:regioninfo/1731476372293/Put/seqid=0 2024-11-13T05:40:09,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741884_1060 (size=5672) 2024-11-13T05:40:09,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741884_1060 (size=5672) 2024-11-13T05:40:09,874 INFO [M:0;a61c20bd729c:35097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/56ce54216b3b41e0bafa1ffc58f0b6c1 2024-11-13T05:40:09,891 DEBUG [M:0;a61c20bd729c:35097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb27d56d8afd4ddabd5b1efffa44fccf is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731476372954/Put/seqid=0 2024-11-13T05:40:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741885_1061 (size=7090) 2024-11-13T05:40:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741885_1061 (size=7090) 2024-11-13T05:40:09,895 INFO [M:0;a61c20bd729c:35097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb27d56d8afd4ddabd5b1efffa44fccf 2024-11-13T05:40:09,900 INFO [M:0;a61c20bd729c:35097 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb27d56d8afd4ddabd5b1efffa44fccf 2024-11-13T05:40:09,914 DEBUG [M:0;a61c20bd729c:35097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33b7050d469441e2adc0df87db0e8cb8 is 69, key is a61c20bd729c,37093,1731476371235/rs:state/1731476371714/Put/seqid=0 2024-11-13T05:40:09,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741886_1062 (size=5156) 2024-11-13T05:40:09,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741886_1062 (size=5156) 2024-11-13T05:40:09,919 INFO [M:0;a61c20bd729c:35097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33b7050d469441e2adc0df87db0e8cb8 2024-11-13T05:40:09,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:09,922 INFO [RS:0;a61c20bd729c:37093 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:40:09,922 INFO [RS:0;a61c20bd729c:37093 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,37093,1731476371235; zookeeper connection closed. 2024-11-13T05:40:09,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37093-0x10132b4b2020001, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:09,922 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a968168 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a968168 2024-11-13T05:40:09,923 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:40:09,937 DEBUG [M:0;a61c20bd729c:35097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2554e8fd85f749af90a2908b4e37c9bf is 52, key is load_balancer_on/state:d/1731476372571/Put/seqid=0 2024-11-13T05:40:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741887_1063 (size=5056) 2024-11-13T05:40:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741887_1063 (size=5056) 2024-11-13T05:40:09,942 INFO [M:0;a61c20bd729c:35097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2554e8fd85f749af90a2908b4e37c9bf 2024-11-13T05:40:09,946 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/56ce54216b3b41e0bafa1ffc58f0b6c1 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/56ce54216b3b41e0bafa1ffc58f0b6c1 2024-11-13T05:40:09,951 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/56ce54216b3b41e0bafa1ffc58f0b6c1, entries=8, sequenceid=125, filesize=5.5 K 2024-11-13T05:40:09,951 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eb27d56d8afd4ddabd5b1efffa44fccf as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eb27d56d8afd4ddabd5b1efffa44fccf 2024-11-13T05:40:09,955 INFO [M:0;a61c20bd729c:35097 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb27d56d8afd4ddabd5b1efffa44fccf 2024-11-13T05:40:09,955 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eb27d56d8afd4ddabd5b1efffa44fccf, entries=13, sequenceid=125, filesize=6.9 K 2024-11-13T05:40:09,956 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33b7050d469441e2adc0df87db0e8cb8 as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33b7050d469441e2adc0df87db0e8cb8 2024-11-13T05:40:09,960 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33b7050d469441e2adc0df87db0e8cb8, entries=1, sequenceid=125, filesize=5.0 K 2024-11-13T05:40:09,961 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2554e8fd85f749af90a2908b4e37c9bf as hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2554e8fd85f749af90a2908b4e37c9bf 2024-11-13T05:40:09,964 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44363/user/jenkins/test-data/677caf6c-ad76-d8a0-c132-4116551d35b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2554e8fd85f749af90a2908b4e37c9bf, entries=1, sequenceid=125, filesize=4.9 K 2024-11-13T05:40:09,965 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=125, compaction requested=false 2024-11-13T05:40:09,966 INFO [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:09,967 DEBUG [M:0;a61c20bd729c:35097 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476409854Disabling compacts and flushes for region at 1731476409854Disabling writes for close at 1731476409854Obtaining lock to block concurrent updates at 1731476409855 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476409855Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1731476409855Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476409855Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476409856 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476409868 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476409868Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476409877 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476409890 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476409890Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476409900 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476409914 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476409914Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476409922 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476409937 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476409937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b9f7d6b: reopening flushed file at 1731476409946 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ba2c0a3: reopening flushed file at 1731476409951 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ce070a4: reopening flushed file at 1731476409955 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b4b5877: reopening flushed file at 1731476409960 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=125, compaction requested=false at 1731476409965 (+5 ms)Writing region close event to WAL at 1731476409966 (+1 ms)Closed at 1731476409966 2024-11-13T05:40:09,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:09,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741830_1006 (size=61332) 2024-11-13T05:40:09,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41287 is added to blk_1073741830_1006 (size=61332) 2024-11-13T05:40:09,969 INFO [M:0;a61c20bd729c:35097 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:40:09,969 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:40:09,969 INFO [M:0;a61c20bd729c:35097 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35097 2024-11-13T05:40:09,969 INFO [M:0;a61c20bd729c:35097 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:40:10,018 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:10,018 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:10,080 INFO [M:0;a61c20bd729c:35097 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:40:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35097-0x10132b4b2020000, quorum=127.0.0.1:49847, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:10,123 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@41fbd5aa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:40:10,123 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2480e20a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:40:10,124 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:40:10,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@adc7967{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:40:10,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@686a5b12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir/,STOPPED} 2024-11-13T05:40:10,127 WARN [BP-300909309-172.17.0.3-1731476368937 heartbeating to localhost/127.0.0.1:44363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:40:10,127 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:40:10,127 WARN [BP-300909309-172.17.0.3-1731476368937 heartbeating to localhost/127.0.0.1:44363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-300909309-172.17.0.3-1731476368937 (Datanode Uuid d2a0943d-ef6d-4d6f-b749-165bcca97e46) service to localhost/127.0.0.1:44363 2024-11-13T05:40:10,127 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:40:10,128 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data3/current/BP-300909309-172.17.0.3-1731476368937 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:10,128 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data4/current/BP-300909309-172.17.0.3-1731476368937 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:10,128 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:40:10,130 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@33cca936{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:40:10,131 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f9d87e6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:40:10,131 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:40:10,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bdfb606{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:40:10,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9bd0638{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir/,STOPPED} 2024-11-13T05:40:10,132 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:40:10,132 WARN [BP-300909309-172.17.0.3-1731476368937 heartbeating to localhost/127.0.0.1:44363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:40:10,132 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:40:10,132 WARN [BP-300909309-172.17.0.3-1731476368937 heartbeating to localhost/127.0.0.1:44363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-300909309-172.17.0.3-1731476368937 (Datanode Uuid 516f1ee5-5475-4f9f-94b1-3a91ac34d3e5) service to localhost/127.0.0.1:44363 2024-11-13T05:40:10,133 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data1/current/BP-300909309-172.17.0.3-1731476368937 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:10,133 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/cluster_4fc11513-4a02-65fe-0b9e-33cd24a4b79c/data/data2/current/BP-300909309-172.17.0.3-1731476368937 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:10,133 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:40:10,138 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@98b6596{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:40:10,139 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a63bfa6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:40:10,139 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:40:10,139 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1cb72b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:40:10,139 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@623cbb81{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir/,STOPPED} 2024-11-13T05:40:10,146 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:40:10,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:40:10,184 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=229 (was 205) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44363 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44363 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44363 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44363 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44363 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44363 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44363 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44363 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=519 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=72 (was 116), ProcessCount=11 (was 11), AvailableMemoryMB=4626 (was 4665) 2024-11-13T05:40:10,191 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=229, OpenFileDescriptor=519, MaxFileDescriptor=1048576, SystemLoadAverage=72, ProcessCount=11, AvailableMemoryMB=4627 2024-11-13T05:40:10,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.log.dir so I do NOT create it in target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/64e99dc8-237c-63f7-4b21-0bff6660e416/hadoop.tmp.dir so I do NOT create it in target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177, deleteOnExit=true 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/test.cache.data in system properties and HBase conf 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:40:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:40:10,192 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:40:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:40:10,206 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:40:10,745 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:40:10,748 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:40:10,749 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:40:10,749 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:40:10,749 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:40:10,749 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:40:10,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b6a009{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:40:10,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7583bc54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:40:10,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5139465a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/java.io.tmpdir/jetty-localhost-43771-hadoop-hdfs-3_4_1-tests_jar-_-any-8132903261041589367/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:40:10,841 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@522b7fb7{HTTP/1.1, (http/1.1)}{localhost:43771} 2024-11-13T05:40:10,841 INFO [Time-limited test {}] server.Server(415): Started @292808ms 2024-11-13T05:40:10,852 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-13T05:40:11,018 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:11,018 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:11,109 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:40:11,112 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:40:11,113 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:40:11,113 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:40:11,113 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:40:11,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fbadfdb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:40:11,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cf6e228{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:40:11,205 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5e7ede3b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/java.io.tmpdir/jetty-localhost-40607-hadoop-hdfs-3_4_1-tests_jar-_-any-15567495947878315606/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:40:11,205 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f8a62a5{HTTP/1.1, (http/1.1)}{localhost:40607} 2024-11-13T05:40:11,205 INFO [Time-limited test {}] server.Server(415): Started @293173ms 2024-11-13T05:40:11,206 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:40:11,229 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:40:11,231 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:40:11,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:40:11,232 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:40:11,232 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:40:11,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef21bcc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:40:11,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b0cfbd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:40:11,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c741137{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/java.io.tmpdir/jetty-localhost-43365-hadoop-hdfs-3_4_1-tests_jar-_-any-6557502860487435953/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:40:11,326 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e7d2ab7{HTTP/1.1, (http/1.1)}{localhost:43365} 2024-11-13T05:40:11,326 INFO [Time-limited test {}] server.Server(415): Started @293294ms 2024-11-13T05:40:11,327 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:40:12,019 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:12,019 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:12,384 WARN [Thread-2510 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data2/current/BP-1317257573-172.17.0.3-1731476410209/current, will proceed with Du for space computation calculation, 2024-11-13T05:40:12,384 WARN [Thread-2509 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data1/current/BP-1317257573-172.17.0.3-1731476410209/current, will proceed with Du for space computation calculation, 2024-11-13T05:40:12,399 WARN [Thread-2473 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:40:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc4103e406eb65ce with lease ID 0x51eebdbc80a42b83: Processing first storage report for DS-74824e0a-582b-4ae7-af44-55a61765bc9f from datanode DatanodeRegistration(127.0.0.1:33405, datanodeUuid=e0aee8aa-dd40-40ae-a946-9df09a17e4a0, infoPort=46523, infoSecurePort=0, ipcPort=37363, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209) 2024-11-13T05:40:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc4103e406eb65ce with lease ID 0x51eebdbc80a42b83: from storage DS-74824e0a-582b-4ae7-af44-55a61765bc9f node DatanodeRegistration(127.0.0.1:33405, datanodeUuid=e0aee8aa-dd40-40ae-a946-9df09a17e4a0, infoPort=46523, infoSecurePort=0, ipcPort=37363, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:40:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc4103e406eb65ce with lease ID 0x51eebdbc80a42b83: Processing first storage report for DS-777ac8f3-b09f-4964-bc93-bfe7bf8efbad from datanode DatanodeRegistration(127.0.0.1:33405, datanodeUuid=e0aee8aa-dd40-40ae-a946-9df09a17e4a0, infoPort=46523, infoSecurePort=0, ipcPort=37363, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209) 2024-11-13T05:40:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc4103e406eb65ce with lease ID 0x51eebdbc80a42b83: from storage DS-777ac8f3-b09f-4964-bc93-bfe7bf8efbad node DatanodeRegistration(127.0.0.1:33405, datanodeUuid=e0aee8aa-dd40-40ae-a946-9df09a17e4a0, infoPort=46523, infoSecurePort=0, ipcPort=37363, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:40:12,531 WARN [Thread-2520 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data3/current/BP-1317257573-172.17.0.3-1731476410209/current, will proceed with Du for space computation calculation, 2024-11-13T05:40:12,531 WARN [Thread-2521 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data4/current/BP-1317257573-172.17.0.3-1731476410209/current, will proceed with Du for space computation calculation, 2024-11-13T05:40:12,551 WARN [Thread-2496 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:40:12,553 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6418afae746a43a5 with lease ID 0x51eebdbc80a42b84: Processing first storage report for DS-b89de5ac-33e1-48ae-935d-bbb4f32a5a72 from datanode DatanodeRegistration(127.0.0.1:36089, datanodeUuid=023eb157-fbf9-4054-9c7d-a1abae644c1a, infoPort=37395, infoSecurePort=0, ipcPort=43925, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209) 2024-11-13T05:40:12,553 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6418afae746a43a5 with lease ID 0x51eebdbc80a42b84: from storage DS-b89de5ac-33e1-48ae-935d-bbb4f32a5a72 node DatanodeRegistration(127.0.0.1:36089, datanodeUuid=023eb157-fbf9-4054-9c7d-a1abae644c1a, infoPort=37395, infoSecurePort=0, ipcPort=43925, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:40:12,553 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6418afae746a43a5 with lease ID 0x51eebdbc80a42b84: Processing first storage report for DS-b047afc0-feea-4c3a-b1cd-4e81ba81ee7a from datanode DatanodeRegistration(127.0.0.1:36089, datanodeUuid=023eb157-fbf9-4054-9c7d-a1abae644c1a, infoPort=37395, infoSecurePort=0, ipcPort=43925, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209) 2024-11-13T05:40:12,553 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6418afae746a43a5 with lease ID 0x51eebdbc80a42b84: from storage DS-b047afc0-feea-4c3a-b1cd-4e81ba81ee7a node DatanodeRegistration(127.0.0.1:36089, datanodeUuid=023eb157-fbf9-4054-9c7d-a1abae644c1a, infoPort=37395, infoSecurePort=0, ipcPort=43925, storageInfo=lv=-57;cid=testClusterID;nsid=1785682407;c=1731476410209), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:40:12,555 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40 2024-11-13T05:40:12,557 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/zookeeper_0, clientPort=58307, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:40:12,558 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58307 2024-11-13T05:40:12,558 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,559 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:40:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:40:12,569 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401 with version=8 2024-11-13T05:40:12,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38293/user/jenkins/test-data/fbe5d6e7-87cd-0f17-18dd-bd8c7ace207c/hbase-staging 2024-11-13T05:40:12,571 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:40:12,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:40:12,572 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33075 2024-11-13T05:40:12,573 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33075 connecting to ZooKeeper ensemble=127.0.0.1:58307 2024-11-13T05:40:12,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:330750x0, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:40:12,635 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33075-0x10132b554380000 connected 2024-11-13T05:40:12,765 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,767 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:40:12,769 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401, hbase.cluster.distributed=false 2024-11-13T05:40:12,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:40:12,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33075 2024-11-13T05:40:12,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33075 2024-11-13T05:40:12,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33075 2024-11-13T05:40:12,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33075 2024-11-13T05:40:12,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33075 2024-11-13T05:40:12,786 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a61c20bd729c:0 server-side Connection retries=45 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:40:12,786 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:40:12,787 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38101 2024-11-13T05:40:12,788 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38101 connecting to ZooKeeper ensemble=127.0.0.1:58307 2024-11-13T05:40:12,788 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,789 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381010x0, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:40:12,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38101-0x10132b554380001 connected 2024-11-13T05:40:12,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:40:12,801 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:40:12,802 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:40:12,803 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:40:12,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:40:12,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38101 2024-11-13T05:40:12,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38101 2024-11-13T05:40:12,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38101 2024-11-13T05:40:12,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38101 2024-11-13T05:40:12,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38101 2024-11-13T05:40:12,820 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a61c20bd729c:33075 2024-11-13T05:40:12,821 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a61c20bd729c,33075,1731476412571 2024-11-13T05:40:12,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:40:12,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:40:12,832 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a61c20bd729c,33075,1731476412571 2024-11-13T05:40:12,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:40:12,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:12,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:12,843 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:40:12,844 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a61c20bd729c,33075,1731476412571 from backup master directory 2024-11-13T05:40:12,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a61c20bd729c,33075,1731476412571 2024-11-13T05:40:12,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:40:12,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:40:12,853 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:40:12,853 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a61c20bd729c,33075,1731476412571 2024-11-13T05:40:12,860 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/hbase.id] with ID: 9cb78e63-77cd-4af1-8beb-a62bd8e5145b 2024-11-13T05:40:12,860 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/.tmp/hbase.id 2024-11-13T05:40:12,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:40:12,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:40:12,868 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/.tmp/hbase.id]:[hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/hbase.id] 2024-11-13T05:40:12,881 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:12,881 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:40:12,882 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-13T05:40:12,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:12,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:40:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:40:12,902 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:40:12,903 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:40:12,903 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:40:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:40:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:40:12,912 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store 2024-11-13T05:40:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:40:12,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:40:12,921 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:12,921 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:40:12,921 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:12,921 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:12,921 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:40:12,921 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:12,921 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:12,921 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476412921Disabling compacts and flushes for region at 1731476412921Disabling writes for close at 1731476412921Writing region close event to WAL at 1731476412921Closed at 1731476412921 2024-11-13T05:40:12,922 WARN [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/.initializing 2024-11-13T05:40:12,922 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/WALs/a61c20bd729c,33075,1731476412571 2024-11-13T05:40:12,925 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C33075%2C1731476412571, suffix=, logDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/WALs/a61c20bd729c,33075,1731476412571, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/oldWALs, maxLogs=10 2024-11-13T05:40:12,926 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C33075%2C1731476412571.1731476412925 2024-11-13T05:40:12,930 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/WALs/a61c20bd729c,33075,1731476412571/a61c20bd729c%2C33075%2C1731476412571.1731476412925 2024-11-13T05:40:12,932 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37395:37395),(127.0.0.1/127.0.0.1:46523:46523)] 2024-11-13T05:40:12,932 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:40:12,933 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:12,933 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,933 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,936 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:40:12,936 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:12,936 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:12,937 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:40:12,938 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:12,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:12,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,940 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:40:12,940 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:12,941 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:12,941 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:40:12,942 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:12,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:12,943 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,944 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,945 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,947 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,947 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,948 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:40:12,949 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:40:12,950 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:40:12,951 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781834, jitterRate=-0.005847513675689697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:40:12,951 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476412933Initializing all the Stores at 1731476412934 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476412934Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476412934Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476412934Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476412934Cleaning up temporary data from old regions at 1731476412947 (+13 ms)Region opened successfully at 1731476412951 (+4 ms) 2024-11-13T05:40:12,951 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:40:12,953 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e94db39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:40:12,954 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:40:12,954 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:40:12,954 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:40:12,954 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:40:12,955 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:40:12,955 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:40:12,955 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:40:12,957 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:40:12,957 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:40:12,969 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:40:12,969 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:40:12,970 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:40:12,979 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:40:12,980 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:40:12,981 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:40:12,990 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:40:12,991 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:40:13,001 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:40:13,005 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:40:13,011 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:40:13,020 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:13,020 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:13,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:40:13,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:40:13,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,022 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a61c20bd729c,33075,1731476412571, sessionid=0x10132b554380000, setting cluster-up flag (Was=false) 2024-11-13T05:40:13,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,074 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:40:13,075 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,33075,1731476412571 2024-11-13T05:40:13,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,127 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:40:13,131 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a61c20bd729c,33075,1731476412571 2024-11-13T05:40:13,134 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:40:13,138 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:40:13,138 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:40:13,139 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:40:13,139 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a61c20bd729c,33075,1731476412571 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a61c20bd729c:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a61c20bd729c:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:40:13,141 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476443142 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:40:13,142 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,142 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:40:13,142 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:40:13,143 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476413143,5,FailOnTimeoutGroup] 2024-11-13T05:40:13,143 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476413143,5,FailOnTimeoutGroup] 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:40:13,143 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,143 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,144 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:40:13,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:40:13,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:40:13,150 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:40:13,150 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401 2024-11-13T05:40:13,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:40:13,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:40:13,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:13,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:40:13,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:40:13,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:40:13,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:40:13,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:40:13,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:40:13,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:40:13,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:40:13,166 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:40:13,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740 2024-11-13T05:40:13,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740 2024-11-13T05:40:13,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:40:13,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:40:13,169 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:40:13,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:40:13,171 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:40:13,172 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=814755, jitterRate=0.036015138030052185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:40:13,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476413157Initializing all the Stores at 1731476413158 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476413158Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476413158Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476413158Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476413158Cleaning up temporary data from old regions at 1731476413168 (+10 ms)Region opened successfully at 1731476413172 (+4 ms) 2024-11-13T05:40:13,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:40:13,172 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:40:13,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:40:13,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:40:13,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:40:13,172 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:40:13,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476413172Disabling compacts and flushes for region at 1731476413172Disabling writes for close at 1731476413172Writing region close event to WAL at 1731476413172Closed at 1731476413172 2024-11-13T05:40:13,173 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:40:13,173 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:40:13,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:40:13,175 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:40:13,176 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:40:13,208 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(746): ClusterId : 9cb78e63-77cd-4af1-8beb-a62bd8e5145b 2024-11-13T05:40:13,209 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:40:13,223 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:40:13,224 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:40:13,235 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:40:13,235 DEBUG [RS:0;a61c20bd729c:38101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf51781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a61c20bd729c/172.17.0.3:0 2024-11-13T05:40:13,251 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a61c20bd729c:38101 2024-11-13T05:40:13,251 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:40:13,251 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:40:13,251 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:40:13,252 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(2659): reportForDuty to master=a61c20bd729c,33075,1731476412571 with port=38101, startcode=1731476412786 2024-11-13T05:40:13,252 DEBUG [RS:0;a61c20bd729c:38101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:40:13,254 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55255, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:40:13,254 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33075 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,254 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33075 {}] master.ServerManager(517): Registering regionserver=a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,255 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401 2024-11-13T05:40:13,255 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37717 2024-11-13T05:40:13,255 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:40:13,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:40:13,264 DEBUG [RS:0;a61c20bd729c:38101 {}] zookeeper.ZKUtil(111): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,264 WARN [RS:0;a61c20bd729c:38101 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:40:13,264 INFO [RS:0;a61c20bd729c:38101 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:40:13,264 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,264 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a61c20bd729c,38101,1731476412786] 2024-11-13T05:40:13,267 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:40:13,268 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:40:13,268 INFO [RS:0;a61c20bd729c:38101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:40:13,268 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,269 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:40:13,269 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:40:13,269 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a61c20bd729c:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a61c20bd729c:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:40:13,270 DEBUG [RS:0;a61c20bd729c:38101 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a61c20bd729c:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:40:13,271 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,271 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,271 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,271 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,271 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,271 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,38101,1731476412786-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:40:13,287 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:40:13,287 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,38101,1731476412786-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,287 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,287 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.Replication(171): a61c20bd729c,38101,1731476412786 started 2024-11-13T05:40:13,299 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,299 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1482): Serving as a61c20bd729c,38101,1731476412786, RpcServer on a61c20bd729c/172.17.0.3:38101, sessionid=0x10132b554380001 2024-11-13T05:40:13,299 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:40:13,299 DEBUG [RS:0;a61c20bd729c:38101 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,299 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,38101,1731476412786' 2024-11-13T05:40:13,299 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a61c20bd729c,38101,1731476412786' 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:40:13,300 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:40:13,301 DEBUG [RS:0;a61c20bd729c:38101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:40:13,301 INFO [RS:0;a61c20bd729c:38101 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:40:13,301 INFO [RS:0;a61c20bd729c:38101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:40:13,326 WARN [a61c20bd729c:33075 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:40:13,405 INFO [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C38101%2C1731476412786, suffix=, logDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/a61c20bd729c,38101,1731476412786, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs, maxLogs=32 2024-11-13T05:40:13,406 INFO [RS:0;a61c20bd729c:38101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C38101%2C1731476412786.1731476413405 2024-11-13T05:40:13,415 INFO [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/a61c20bd729c,38101,1731476412786/a61c20bd729c%2C38101%2C1731476412786.1731476413405 2024-11-13T05:40:13,416 DEBUG [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37395:37395),(127.0.0.1/127.0.0.1:46523:46523)] 2024-11-13T05:40:13,576 DEBUG [a61c20bd729c:33075 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:40:13,577 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,580 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,38101,1731476412786, state=OPENING 2024-11-13T05:40:13,622 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:40:13,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,635 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:40:13,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:40:13,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:40:13,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,38101,1731476412786}] 2024-11-13T05:40:13,790 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:40:13,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:40:13,800 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:40:13,801 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:40:13,803 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a61c20bd729c%2C38101%2C1731476412786.meta, suffix=.meta, logDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/a61c20bd729c,38101,1731476412786, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs, maxLogs=32 2024-11-13T05:40:13,804 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor a61c20bd729c%2C38101%2C1731476412786.meta.1731476413803.meta 2024-11-13T05:40:13,808 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/a61c20bd729c,38101,1731476412786/a61c20bd729c%2C38101%2C1731476412786.meta.1731476413803.meta 2024-11-13T05:40:13,811 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46523:46523),(127.0.0.1/127.0.0.1:37395:37395)] 2024-11-13T05:40:13,813 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:40:13,814 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:40:13,814 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:40:13,814 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:40:13,814 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:40:13,814 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:13,814 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:40:13,814 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:40:13,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:40:13,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:40:13,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:40:13,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:40:13,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:40:13,818 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:40:13,818 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:40:13,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:40:13,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:13,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:40:13,820 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:40:13,820 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740 2024-11-13T05:40:13,821 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740 2024-11-13T05:40:13,823 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:40:13,823 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:40:13,823 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:40:13,824 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:40:13,825 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=733774, jitterRate=-0.06695815920829773}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:40:13,825 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:40:13,825 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476413814Writing region info on filesystem at 1731476413814Initializing all the Stores at 1731476413815 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476413815Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476413815Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476413815Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476413815Cleaning up temporary data from old regions at 1731476413823 (+8 ms)Running coprocessor post-open hooks at 1731476413825 (+2 ms)Region opened successfully at 1731476413825 2024-11-13T05:40:13,826 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476413790 2024-11-13T05:40:13,828 DEBUG [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:40:13,828 INFO [RS_OPEN_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:40:13,829 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,830 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a61c20bd729c,38101,1731476412786, state=OPEN 2024-11-13T05:40:13,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:40:13,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:40:13,869 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:40:13,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:40:13,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:40:13,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a61c20bd729c,38101,1731476412786 in 234 msec 2024-11-13T05:40:13,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:40:13,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 698 msec 2024-11-13T05:40:13,876 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:40:13,876 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:40:13,878 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:13,878 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,38101,1731476412786, seqNum=-1] 2024-11-13T05:40:13,878 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:13,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:13,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 749 msec 2024-11-13T05:40:13,887 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476413887, completionTime=-1 2024-11-13T05:40:13,887 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:40:13,887 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T05:40:13,890 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T05:40:13,890 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476473890 2024-11-13T05:40:13,890 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476533890 2024-11-13T05:40:13,890 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec 2024-11-13T05:40:13,891 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,33075,1731476412571-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,891 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,33075,1731476412571-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,891 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,33075,1731476412571-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,891 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a61c20bd729c:33075, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,891 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,891 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,893 DEBUG [master/a61c20bd729c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:40:13,896 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.043sec 2024-11-13T05:40:13,896 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:40:13,896 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:40:13,896 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:40:13,897 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:40:13,897 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:40:13,897 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,33075,1731476412571-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:40:13,897 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,33075,1731476412571-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:40:13,900 DEBUG [master/a61c20bd729c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:40:13,900 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:40:13,900 INFO [master/a61c20bd729c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a61c20bd729c,33075,1731476412571-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:40:13,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c1f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:13,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a61c20bd729c,33075,-1 for getting cluster id 2024-11-13T05:40:13,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:13,910 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9cb78e63-77cd-4af1-8beb-a62bd8e5145b' 2024-11-13T05:40:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9cb78e63-77cd-4af1-8beb-a62bd8e5145b" 2024-11-13T05:40:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3bba38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a61c20bd729c,33075,-1] 2024-11-13T05:40:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:13,913 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:13,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dbaf83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:13,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a61c20bd729c,38101,1731476412786, seqNum=-1] 2024-11-13T05:40:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:13,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:13,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a61c20bd729c,33075,1731476412571 2024-11-13T05:40:13,917 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:40:13,920 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-13T05:40:13,920 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-13T05:40:13,923 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/test.com,8080,1, archiveDir=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs, maxLogs=32 2024-11-13T05:40:13,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731476413924 2024-11-13T05:40:13,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/test.com,8080,1/test.com%2C8080%2C1.1731476413924 2024-11-13T05:40:13,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46523:46523),(127.0.0.1/127.0.0.1:37395:37395)] 2024-11-13T05:40:13,936 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731476413936 2024-11-13T05:40:13,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,942 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/test.com,8080,1/test.com%2C8080%2C1.1731476413924 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/test.com,8080,1/test.com%2C8080%2C1.1731476413936 2024-11-13T05:40:13,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46523:46523),(127.0.0.1/127.0.0.1:37395:37395)] 2024-11-13T05:40:13,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/test.com,8080,1/test.com%2C8080%2C1.1731476413924 is not closed yet, will try archiving it next time 2024-11-13T05:40:13,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741835_1011 (size=93) 2024-11-13T05:40:13,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741835_1011 (size=93) 2024-11-13T05:40:13,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:13,945 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/WALs/test.com,8080,1/test.com%2C8080%2C1.1731476413924 to hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs/test.com%2C8080%2C1.1731476413924 2024-11-13T05:40:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741836_1012 (size=93) 2024-11-13T05:40:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741836_1012 (size=93) 2024-11-13T05:40:13,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs 2024-11-13T05:40:13,951 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731476413936) 2024-11-13T05:40:13,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:40:13,951 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:40:13,951 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:13,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:13,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:13,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:13,952 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:40:13,952 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=510083689, stopped=false 2024-11-13T05:40:13,952 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a61c20bd729c,33075,1731476412571 2024-11-13T05:40:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:40:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:40:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:13,997 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:40:13,997 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:40:13,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:13,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:13,997 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a61c20bd729c,38101,1731476412786' ***** 2024-11-13T05:40:13,997 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:40:13,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:40:13,998 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(959): stopping server a61c20bd729c,38101,1731476412786 2024-11-13T05:40:13,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a61c20bd729c:38101. 2024-11-13T05:40:13,998 DEBUG [RS:0;a61c20bd729c:38101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:13,998 DEBUG [RS:0;a61c20bd729c:38101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:40:13,998 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:40:13,999 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:40:13,999 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:40:13,999 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T05:40:13,999 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:40:13,999 DEBUG [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:40:13,999 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:40:13,999 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:40:13,999 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:40:13,999 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:40:13,999 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:40:13,999 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-13T05:40:14,017 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/.tmp/ns/2b7d8fe4c0b74390b889a7fb8017582a is 43, key is default/ns:d/1731476413880/Put/seqid=0 2024-11-13T05:40:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741837_1013 (size=5153) 2024-11-13T05:40:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741837_1013 (size=5153) 2024-11-13T05:40:14,021 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,44475,1731476228646/a61c20bd729c%2C44475%2C1731476228646.meta.1731476230046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:14,021 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:41149/user/jenkins/test-data/39b1dc4f-2d92-b759-08d3-193313388817/WALs/a61c20bd729c,35337,1731476230212/a61c20bd729c%2C35337%2C1731476230212.1731476230452 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-13T05:40:14,021 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/.tmp/ns/2b7d8fe4c0b74390b889a7fb8017582a 2024-11-13T05:40:14,026 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/.tmp/ns/2b7d8fe4c0b74390b889a7fb8017582a as hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/ns/2b7d8fe4c0b74390b889a7fb8017582a 2024-11-13T05:40:14,032 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/ns/2b7d8fe4c0b74390b889a7fb8017582a, entries=2, sequenceid=6, filesize=5.0 K 2024-11-13T05:40:14,033 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=6, compaction requested=false 2024-11-13T05:40:14,033 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:40:14,038 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T05:40:14,039 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:40:14,039 INFO [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:40:14,039 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476413999Running coprocessor pre-close hooks at 1731476413999Disabling compacts and flushes for region at 1731476413999Disabling writes for close at 1731476413999Obtaining lock to block concurrent updates at 1731476413999Preparing flush snapshotting stores in 1588230740 at 1731476413999Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731476414000 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731476414001 (+1 ms)Flushing 1588230740/ns: creating writer at 1731476414001Flushing 1588230740/ns: appending metadata at 1731476414017 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731476414017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b274aae: reopening flushed file at 1731476414025 (+8 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=6, compaction requested=false at 1731476414033 (+8 ms)Writing region close event to WAL at 1731476414035 (+2 ms)Running coprocessor post-close hooks at 1731476414039 (+4 ms)Closed at 1731476414039 2024-11-13T05:40:14,039 DEBUG [RS_CLOSE_META-regionserver/a61c20bd729c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:40:14,199 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(976): stopping server a61c20bd729c,38101,1731476412786; all regions closed. 2024-11-13T05:40:14,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741834_1010 (size=1152) 2024-11-13T05:40:14,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741834_1010 (size=1152) 2024-11-13T05:40:14,209 DEBUG [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs 2024-11-13T05:40:14,209 INFO [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C38101%2C1731476412786.meta:.meta(num 1731476413803) 2024-11-13T05:40:14,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741833_1009 (size=93) 2024-11-13T05:40:14,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741833_1009 (size=93) 2024-11-13T05:40:14,215 DEBUG [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/oldWALs 2024-11-13T05:40:14,215 INFO [RS:0;a61c20bd729c:38101 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog a61c20bd729c%2C38101%2C1731476412786:(num 1731476413405) 2024-11-13T05:40:14,215 DEBUG [RS:0;a61c20bd729c:38101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,215 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:40:14,215 INFO [RS:0;a61c20bd729c:38101 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:40:14,215 INFO [RS:0;a61c20bd729c:38101 {}] hbase.ChoreService(370): Chore service for: regionserver/a61c20bd729c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:40:14,216 INFO [RS:0;a61c20bd729c:38101 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:40:14,216 INFO [regionserver/a61c20bd729c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:40:14,216 INFO [RS:0;a61c20bd729c:38101 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38101 2024-11-13T05:40:14,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a61c20bd729c,38101,1731476412786 2024-11-13T05:40:14,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:40:14,243 INFO [RS:0;a61c20bd729c:38101 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:40:14,253 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a61c20bd729c,38101,1731476412786] 2024-11-13T05:40:14,264 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a61c20bd729c,38101,1731476412786 already deleted, retry=false 2024-11-13T05:40:14,264 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a61c20bd729c,38101,1731476412786 expired; onlineServers=0 2024-11-13T05:40:14,264 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a61c20bd729c,33075,1731476412571' ***** 2024-11-13T05:40:14,264 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:40:14,264 INFO [M:0;a61c20bd729c:33075 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:40:14,264 INFO [M:0;a61c20bd729c:33075 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:40:14,265 DEBUG [M:0;a61c20bd729c:33075 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:40:14,265 DEBUG [M:0;a61c20bd729c:33075 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:40:14,265 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:40:14,265 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476413143 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.small.0-1731476413143,5,FailOnTimeoutGroup] 2024-11-13T05:40:14,265 DEBUG [master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476413143 {}] cleaner.HFileCleaner(306): Exit Thread[master/a61c20bd729c:0:becomeActiveMaster-HFileCleaner.large.0-1731476413143,5,FailOnTimeoutGroup] 2024-11-13T05:40:14,265 INFO [M:0;a61c20bd729c:33075 {}] hbase.ChoreService(370): Chore service for: master/a61c20bd729c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:40:14,265 INFO [M:0;a61c20bd729c:33075 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:40:14,266 DEBUG [M:0;a61c20bd729c:33075 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:40:14,266 INFO [M:0;a61c20bd729c:33075 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:40:14,266 INFO [M:0;a61c20bd729c:33075 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:40:14,266 INFO [M:0;a61c20bd729c:33075 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:40:14,266 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:40:14,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:40:14,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:40:14,275 DEBUG [M:0;a61c20bd729c:33075 {}] zookeeper.ZKUtil(347): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:40:14,275 WARN [M:0;a61c20bd729c:33075 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:40:14,276 INFO [M:0;a61c20bd729c:33075 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/.lastflushedseqids 2024-11-13T05:40:14,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741838_1014 (size=99) 2024-11-13T05:40:14,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741838_1014 (size=99) 2024-11-13T05:40:14,286 INFO [M:0;a61c20bd729c:33075 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:40:14,286 INFO [M:0;a61c20bd729c:33075 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:40:14,286 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:40:14,286 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:14,286 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:14,286 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:40:14,286 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:14,286 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-13T05:40:14,301 DEBUG [M:0;a61c20bd729c:33075 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5930544a2fb3449492183bf0ec94bae5 is 82, key is hbase:meta,,1/info:regioninfo/1731476413829/Put/seqid=0 2024-11-13T05:40:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741839_1015 (size=5672) 2024-11-13T05:40:14,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741839_1015 (size=5672) 2024-11-13T05:40:14,306 INFO [M:0;a61c20bd729c:33075 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5930544a2fb3449492183bf0ec94bae5 2024-11-13T05:40:14,323 DEBUG [M:0;a61c20bd729c:33075 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c18134beb2e64d2ea7c0e52b1d350d19 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731476413886/Put/seqid=0 2024-11-13T05:40:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741840_1016 (size=5275) 2024-11-13T05:40:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741840_1016 (size=5275) 2024-11-13T05:40:14,328 INFO [M:0;a61c20bd729c:33075 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c18134beb2e64d2ea7c0e52b1d350d19 2024-11-13T05:40:14,346 DEBUG [M:0;a61c20bd729c:33075 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0a7231976344921b0dc0427b61ede93 is 69, key is a61c20bd729c,38101,1731476412786/rs:state/1731476413254/Put/seqid=0 2024-11-13T05:40:14,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741841_1017 (size=5156) 2024-11-13T05:40:14,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741841_1017 (size=5156) 2024-11-13T05:40:14,350 INFO [M:0;a61c20bd729c:33075 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0a7231976344921b0dc0427b61ede93 2024-11-13T05:40:14,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:14,353 INFO [RS:0;a61c20bd729c:38101 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:40:14,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38101-0x10132b554380001, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:14,354 INFO [RS:0;a61c20bd729c:38101 {}] regionserver.HRegionServer(1031): Exiting; stopping=a61c20bd729c,38101,1731476412786; zookeeper connection closed. 2024-11-13T05:40:14,354 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36ef13ed {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36ef13ed 2024-11-13T05:40:14,354 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:40:14,369 DEBUG [M:0;a61c20bd729c:33075 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2882e7b889d7411b8ec668702b00b7b6 is 52, key is load_balancer_on/state:d/1731476413919/Put/seqid=0 2024-11-13T05:40:14,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741842_1018 (size=5056) 2024-11-13T05:40:14,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741842_1018 (size=5056) 2024-11-13T05:40:14,374 INFO [M:0;a61c20bd729c:33075 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2882e7b889d7411b8ec668702b00b7b6 2024-11-13T05:40:14,377 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5930544a2fb3449492183bf0ec94bae5 as hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5930544a2fb3449492183bf0ec94bae5 2024-11-13T05:40:14,381 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5930544a2fb3449492183bf0ec94bae5, entries=8, sequenceid=29, filesize=5.5 K 2024-11-13T05:40:14,382 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c18134beb2e64d2ea7c0e52b1d350d19 as hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c18134beb2e64d2ea7c0e52b1d350d19 2024-11-13T05:40:14,386 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c18134beb2e64d2ea7c0e52b1d350d19, entries=3, sequenceid=29, filesize=5.2 K 2024-11-13T05:40:14,387 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0a7231976344921b0dc0427b61ede93 as hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0a7231976344921b0dc0427b61ede93 2024-11-13T05:40:14,391 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0a7231976344921b0dc0427b61ede93, entries=1, sequenceid=29, filesize=5.0 K 2024-11-13T05:40:14,391 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2882e7b889d7411b8ec668702b00b7b6 as hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2882e7b889d7411b8ec668702b00b7b6 2024-11-13T05:40:14,395 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37717/user/jenkins/test-data/9e0f5921-b018-0c93-0f6b-01e4e2d6a401/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2882e7b889d7411b8ec668702b00b7b6, entries=1, sequenceid=29, filesize=4.9 K 2024-11-13T05:40:14,396 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=29, compaction requested=false 2024-11-13T05:40:14,397 INFO [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:40:14,397 DEBUG [M:0;a61c20bd729c:33075 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476414286Disabling compacts and flushes for region at 1731476414286Disabling writes for close at 1731476414286Obtaining lock to block concurrent updates at 1731476414286Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476414286Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731476414286Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476414287 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476414287Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476414301 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476414301Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476414310 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476414322 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476414322Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476414331 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476414345 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476414345Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731476414354 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731476414369 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731476414369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@338a80d5: reopening flushed file at 1731476414377 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c0e077a: reopening flushed file at 1731476414381 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5814729c: reopening flushed file at 1731476414386 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77f5ee82: reopening flushed file at 1731476414391 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=29, compaction requested=false at 1731476414396 (+5 ms)Writing region close event to WAL at 1731476414397 (+1 ms)Closed at 1731476414397 2024-11-13T05:40:14,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:40:14,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33405 is added to blk_1073741830_1006 (size=10311) 2024-11-13T05:40:14,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36089 is added to blk_1073741830_1006 (size=10311) 2024-11-13T05:40:14,400 INFO [M:0;a61c20bd729c:33075 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:40:14,400 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:40:14,400 INFO [M:0;a61c20bd729c:33075 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33075 2024-11-13T05:40:14,400 INFO [M:0;a61c20bd729c:33075 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:40:14,533 INFO [M:0;a61c20bd729c:33075 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:40:14,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:14,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33075-0x10132b554380000, quorum=127.0.0.1:58307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:40:14,551 WARN [BP-1317257573-172.17.0.3-1731476410209 heartbeating to localhost/127.0.0.1:37717 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1317257573-172.17.0.3-1731476410209 (Datanode Uuid 023eb157-fbf9-4054-9c7d-a1abae644c1a) service to localhost/127.0.0.1:37717 2024-11-13T05:40:14,565 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data3/current/BP-1317257573-172.17.0.3-1731476410209 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:14,565 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data4/current/BP-1317257573-172.17.0.3-1731476410209 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:14,567 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c741137{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:40:14,568 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e7d2ab7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:40:14,568 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:40:14,569 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b0cfbd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:40:14,569 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef21bcc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir/,STOPPED} 2024-11-13T05:40:14,571 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:40:14,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5e7ede3b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:40:14,574 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f8a62a5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:40:14,574 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:40:14,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cf6e228{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:40:14,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fbadfdb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir/,STOPPED} 2024-11-13T05:40:14,575 WARN [BP-1317257573-172.17.0.3-1731476410209 heartbeating to localhost/127.0.0.1:37717 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:40:14,575 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:40:14,575 WARN [BP-1317257573-172.17.0.3-1731476410209 heartbeating to localhost/127.0.0.1:37717 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1317257573-172.17.0.3-1731476410209 (Datanode Uuid e0aee8aa-dd40-40ae-a946-9df09a17e4a0) service to localhost/127.0.0.1:37717 2024-11-13T05:40:14,575 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:40:14,576 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data1/current/BP-1317257573-172.17.0.3-1731476410209 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:14,576 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/cluster_94dc8a17-2ddc-069d-bf90-cdeb9cd99177/data/data2/current/BP-1317257573-172.17.0.3-1731476410209 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:40:14,576 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:40:14,581 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5139465a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:40:14,582 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@522b7fb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:40:14,582 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:40:14,582 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7583bc54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:40:14,582 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b6a009{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0fb1800b-8fb0-6564-7080-0726015b0c40/hadoop.log.dir/,STOPPED} 2024-11-13T05:40:14,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,588 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:40:14,588 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,588 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,588 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,588 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,589 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,589 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,590 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,590 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:40:14,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:40:14,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 229) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37717 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:37717 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37717 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37717 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:37717 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:37717 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37717 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Command processor java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411) app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:37717 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=534 (was 519) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=82 (was 72) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4619 (was 4627) 2024-11-13T05:40:14,612 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null