2024-11-23 02:40:10,995 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-23 02:40:11,014 main DEBUG Took 0.010062 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 02:40:11,015 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 02:40:11,015 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 02:40:11,016 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 02:40:11,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,024 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 02:40:11,035 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,037 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,038 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,039 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,039 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,040 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,040 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,041 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,042 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,042 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,042 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,043 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,043 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,044 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,045 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,045 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,045 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 02:40:11,046 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,046 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 02:40:11,048 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 02:40:11,049 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 02:40:11,051 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 02:40:11,051 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 02:40:11,052 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 02:40:11,052 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 02:40:11,064 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 02:40:11,067 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 02:40:11,069 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 02:40:11,070 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 02:40:11,070 main DEBUG createAppenders(={Console}) 2024-11-23 02:40:11,071 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-23 02:40:11,072 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-23 02:40:11,072 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-23 02:40:11,073 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 02:40:11,073 main DEBUG OutputStream closed 2024-11-23 02:40:11,074 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 02:40:11,074 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 02:40:11,074 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-23 02:40:11,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 02:40:11,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 02:40:11,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 02:40:11,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 02:40:11,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 02:40:11,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 02:40:11,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 02:40:11,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 02:40:11,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 02:40:11,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 02:40:11,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 02:40:11,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 02:40:11,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 02:40:11,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 02:40:11,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 02:40:11,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 02:40:11,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 02:40:11,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 02:40:11,169 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 02:40:11,169 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-23 02:40:11,170 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 02:40:11,170 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-23T02:40:11,184 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-23 02:40:11,186 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 02:40:11,187 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T02:40:11,490 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797 2024-11-23T02:40:11,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T02:40:11,525 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e, deleteOnExit=true 2024-11-23T02:40:11,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T02:40:11,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/test.cache.data in system properties and HBase conf 2024-11-23T02:40:11,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T02:40:11,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/hadoop.log.dir in system properties and HBase conf 2024-11-23T02:40:11,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T02:40:11,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T02:40:11,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T02:40:11,648 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T02:40:11,749 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T02:40:11,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T02:40:11,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T02:40:11,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T02:40:11,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T02:40:11,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T02:40:11,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T02:40:11,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T02:40:11,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T02:40:11,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T02:40:11,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/nfs.dump.dir in system properties and HBase conf 2024-11-23T02:40:11,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/java.io.tmpdir in system properties and HBase conf 2024-11-23T02:40:11,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T02:40:11,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T02:40:11,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T02:40:12,700 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T02:40:12,794 INFO [Time-limited test {}] log.Log(170): Logging initialized @2616ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T02:40:12,886 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T02:40:12,962 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T02:40:12,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T02:40:12,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T02:40:12,987 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T02:40:13,001 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T02:40:13,003 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/hadoop.log.dir/,AVAILABLE} 2024-11-23T02:40:13,004 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T02:40:13,206 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/java.io.tmpdir/jetty-localhost-33991-hadoop-hdfs-3_4_1-tests_jar-_-any-17382013128254353675/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T02:40:13,216 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:33991} 2024-11-23T02:40:13,217 INFO [Time-limited test {}] server.Server(415): Started @3040ms 2024-11-23T02:40:13,638 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T02:40:13,645 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T02:40:13,646 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T02:40:13,646 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T02:40:13,646 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T02:40:13,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/hadoop.log.dir/,AVAILABLE} 2024-11-23T02:40:13,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T02:40:13,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/java.io.tmpdir/jetty-localhost-33437-hadoop-hdfs-3_4_1-tests_jar-_-any-1016355034229748340/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T02:40:13,784 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:33437} 2024-11-23T02:40:13,784 INFO [Time-limited test {}] server.Server(415): Started @3608ms 2024-11-23T02:40:13,853 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T02:40:14,396 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/data/data2/current/BP-1046908168-172.17.0.2-1732329612428/current, will proceed with Du for space computation calculation, 2024-11-23T02:40:14,396 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/data/data1/current/BP-1046908168-172.17.0.2-1732329612428/current, will proceed with Du for space computation calculation, 2024-11-23T02:40:14,440 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T02:40:14,504 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf697c7e9cc5928c6 with lease ID 0xedba8687422d36da: Processing first storage report for DS-ebc9641e-4e9b-496b-8a11-6e62876b6f42 from datanode DatanodeRegistration(127.0.0.1:45173, datanodeUuid=ad12032d-151d-44fa-9769-8468792ac767, infoPort=33259, infoSecurePort=0, ipcPort=43345, storageInfo=lv=-57;cid=testClusterID;nsid=21275146;c=1732329612428) 2024-11-23T02:40:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf697c7e9cc5928c6 with lease ID 0xedba8687422d36da: from storage DS-ebc9641e-4e9b-496b-8a11-6e62876b6f42 node DatanodeRegistration(127.0.0.1:45173, datanodeUuid=ad12032d-151d-44fa-9769-8468792ac767, infoPort=33259, infoSecurePort=0, ipcPort=43345, storageInfo=lv=-57;cid=testClusterID;nsid=21275146;c=1732329612428), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T02:40:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf697c7e9cc5928c6 with lease ID 0xedba8687422d36da: Processing first storage report for DS-d13c526d-8d82-49c6-b91a-faaba20850c0 from datanode DatanodeRegistration(127.0.0.1:45173, datanodeUuid=ad12032d-151d-44fa-9769-8468792ac767, infoPort=33259, infoSecurePort=0, ipcPort=43345, storageInfo=lv=-57;cid=testClusterID;nsid=21275146;c=1732329612428) 2024-11-23T02:40:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf697c7e9cc5928c6 with lease ID 0xedba8687422d36da: from storage DS-d13c526d-8d82-49c6-b91a-faaba20850c0 node DatanodeRegistration(127.0.0.1:45173, datanodeUuid=ad12032d-151d-44fa-9769-8468792ac767, infoPort=33259, infoSecurePort=0, ipcPort=43345, storageInfo=lv=-57;cid=testClusterID;nsid=21275146;c=1732329612428), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T02:40:14,560 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797 2024-11-23T02:40:14,641 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/zookeeper_0, clientPort=64605, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T02:40:14,651 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64605 2024-11-23T02:40:14,662 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:14,665 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:14,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741825_1001 (size=7) 2024-11-23T02:40:15,296 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e with version=8 2024-11-23T02:40:15,296 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/hbase-staging 2024-11-23T02:40:15,396 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T02:40:15,672 INFO [Time-limited test {}] client.ConnectionUtils(128): master/72b6db343a5d:0 server-side Connection retries=9 2024-11-23T02:40:15,684 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T02:40:15,685 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T02:40:15,689 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T02:40:15,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T02:40:15,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T02:40:15,830 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T02:40:15,891 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T02:40:15,900 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T02:40:15,904 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T02:40:15,930 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 702 (auto-detected) 2024-11-23T02:40:15,931 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T02:40:15,951 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43697 2024-11-23T02:40:15,980 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43697 connecting to ZooKeeper ensemble=127.0.0.1:64605 2024-11-23T02:40:16,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436970x0, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T02:40:16,017 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43697-0x10142b217ae0000 connected 2024-11-23T02:40:16,055 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:16,057 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:16,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T02:40:16,073 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e, hbase.cluster.distributed=false 2024-11-23T02:40:16,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T02:40:16,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43697 2024-11-23T02:40:16,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43697 2024-11-23T02:40:16,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43697 2024-11-23T02:40:16,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43697 2024-11-23T02:40:16,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43697 2024-11-23T02:40:16,235 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/72b6db343a5d:0 server-side Connection retries=9 2024-11-23T02:40:16,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T02:40:16,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T02:40:16,237 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T02:40:16,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T02:40:16,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T02:40:16,240 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T02:40:16,242 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T02:40:16,243 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43847 2024-11-23T02:40:16,245 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43847 connecting to ZooKeeper ensemble=127.0.0.1:64605 2024-11-23T02:40:16,247 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:16,251 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:16,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438470x0, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T02:40:16,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43847-0x10142b217ae0001 connected 2024-11-23T02:40:16,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T02:40:16,266 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T02:40:16,274 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T02:40:16,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T02:40:16,282 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T02:40:16,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43847 2024-11-23T02:40:16,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43847 2024-11-23T02:40:16,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43847 2024-11-23T02:40:16,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43847 2024-11-23T02:40:16,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43847 2024-11-23T02:40:16,301 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;72b6db343a5d:43697 2024-11-23T02:40:16,302 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/72b6db343a5d,43697,1732329615460 2024-11-23T02:40:16,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T02:40:16,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T02:40:16,311 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/72b6db343a5d,43697,1732329615460 2024-11-23T02:40:16,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:16,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T02:40:16,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:16,334 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T02:40:16,335 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/72b6db343a5d,43697,1732329615460 from backup master directory 2024-11-23T02:40:16,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/72b6db343a5d,43697,1732329615460 2024-11-23T02:40:16,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T02:40:16,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T02:40:16,338 WARN [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T02:40:16,339 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=72b6db343a5d,43697,1732329615460 2024-11-23T02:40:16,341 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T02:40:16,343 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T02:40:16,407 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/hbase.id] with ID: 49234b28-a0f4-4e16-8eda-c1b946865dbc 2024-11-23T02:40:16,407 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/.tmp/hbase.id 2024-11-23T02:40:16,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741826_1002 (size=42) 2024-11-23T02:40:16,820 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/.tmp/hbase.id]:[hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/hbase.id] 2024-11-23T02:40:16,865 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T02:40:16,872 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T02:40:16,899 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-23T02:40:16,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:16,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:16,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741827_1003 (size=196) 2024-11-23T02:40:17,334 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:40:17,338 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T02:40:17,356 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T02:40:17,360 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T02:40:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741828_1004 (size=1189) 2024-11-23T02:40:17,811 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store 2024-11-23T02:40:17,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741829_1005 (size=34) 2024-11-23T02:40:17,835 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T02:40:17,838 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:17,839 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T02:40:17,839 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:40:17,839 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:40:17,840 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T02:40:17,841 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:40:17,841 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:40:17,842 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732329617839Disabling compacts and flushes for region at 1732329617839Disabling writes for close at 1732329617841 (+2 ms)Writing region close event to WAL at 1732329617841Closed at 1732329617841 2024-11-23T02:40:17,844 WARN [master/72b6db343a5d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/.initializing 2024-11-23T02:40:17,844 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/WALs/72b6db343a5d,43697,1732329615460 2024-11-23T02:40:17,853 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T02:40:17,869 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=72b6db343a5d%2C43697%2C1732329615460, suffix=, logDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/WALs/72b6db343a5d,43697,1732329615460, archiveDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/oldWALs, maxLogs=10 2024-11-23T02:40:17,893 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/WALs/72b6db343a5d,43697,1732329615460/72b6db343a5d%2C43697%2C1732329615460.1732329617873, exclude list is [], retry=0 2024-11-23T02:40:17,910 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45173,DS-ebc9641e-4e9b-496b-8a11-6e62876b6f42,DISK] 2024-11-23T02:40:17,913 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T02:40:17,949 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/WALs/72b6db343a5d,43697,1732329615460/72b6db343a5d%2C43697%2C1732329615460.1732329617873 2024-11-23T02:40:17,950 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33259:33259)] 2024-11-23T02:40:17,950 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:40:17,951 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:17,954 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:17,955 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:17,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T02:40:18,023 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:18,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:18,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,029 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T02:40:18,030 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:18,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:40:18,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T02:40:18,034 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:18,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:40:18,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,037 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T02:40:18,038 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:18,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:40:18,039 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,042 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,043 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,049 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,049 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,053 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T02:40:18,056 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T02:40:18,060 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:40:18,061 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69134267, jitterRate=0.030180856585502625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T02:40:18,067 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732329617968Initializing all the Stores at 1732329617970 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329617971 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329617971Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329617971Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329617972 (+1 ms)Cleaning up temporary data from old regions at 1732329618049 (+77 ms)Region opened successfully at 1732329618067 (+18 ms) 2024-11-23T02:40:18,069 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T02:40:18,102 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cdc5e83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=72b6db343a5d/172.17.0.2:0 2024-11-23T02:40:18,134 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T02:40:18,146 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T02:40:18,146 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T02:40:18,149 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T02:40:18,151 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T02:40:18,156 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T02:40:18,156 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T02:40:18,181 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T02:40:18,190 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T02:40:18,192 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T02:40:18,194 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T02:40:18,196 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T02:40:18,199 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T02:40:18,201 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T02:40:18,205 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T02:40:18,207 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T02:40:18,208 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T02:40:18,210 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T02:40:18,234 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T02:40:18,236 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T02:40:18,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T02:40:18,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T02:40:18,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:18,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:18,245 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=72b6db343a5d,43697,1732329615460, sessionid=0x10142b217ae0000, setting cluster-up flag (Was=false) 2024-11-23T02:40:18,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:18,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:18,266 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T02:40:18,268 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=72b6db343a5d,43697,1732329615460 2024-11-23T02:40:18,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:18,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:18,285 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T02:40:18,287 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=72b6db343a5d,43697,1732329615460 2024-11-23T02:40:18,295 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T02:40:18,373 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T02:40:18,383 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T02:40:18,391 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T02:40:18,393 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(746): ClusterId : 49234b28-a0f4-4e16-8eda-c1b946865dbc 2024-11-23T02:40:18,397 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T02:40:18,398 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 72b6db343a5d,43697,1732329615460 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T02:40:18,403 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T02:40:18,403 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T02:40:18,407 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T02:40:18,407 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/72b6db343a5d:0, corePoolSize=5, maxPoolSize=5 2024-11-23T02:40:18,407 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/72b6db343a5d:0, corePoolSize=5, maxPoolSize=5 2024-11-23T02:40:18,408 DEBUG [RS:0;72b6db343a5d:43847 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442374f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=72b6db343a5d/172.17.0.2:0 2024-11-23T02:40:18,408 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/72b6db343a5d:0, corePoolSize=5, maxPoolSize=5 2024-11-23T02:40:18,408 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/72b6db343a5d:0, corePoolSize=5, maxPoolSize=5 2024-11-23T02:40:18,408 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/72b6db343a5d:0, corePoolSize=10, maxPoolSize=10 2024-11-23T02:40:18,408 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,408 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/72b6db343a5d:0, corePoolSize=2, maxPoolSize=2 2024-11-23T02:40:18,409 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,413 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732329648413 2024-11-23T02:40:18,415 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T02:40:18,416 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T02:40:18,416 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T02:40:18,417 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T02:40:18,420 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T02:40:18,420 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T02:40:18,420 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T02:40:18,421 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T02:40:18,421 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,423 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:18,423 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T02:40:18,427 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;72b6db343a5d:43847 2024-11-23T02:40:18,431 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T02:40:18,432 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T02:40:18,432 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T02:40:18,432 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T02:40:18,432 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T02:40:18,433 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T02:40:18,435 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T02:40:18,435 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T02:40:18,436 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(2659): reportForDuty to master=72b6db343a5d,43697,1732329615460 with port=43847, startcode=1732329616195 2024-11-23T02:40:18,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741831_1007 (size=1321) 2024-11-23T02:40:18,438 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/72b6db343a5d:0:becomeActiveMaster-HFileCleaner.large.0-1732329618437,5,FailOnTimeoutGroup] 2024-11-23T02:40:18,439 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/72b6db343a5d:0:becomeActiveMaster-HFileCleaner.small.0-1732329618438,5,FailOnTimeoutGroup] 2024-11-23T02:40:18,439 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,439 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T02:40:18,440 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,440 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,451 DEBUG [RS:0;72b6db343a5d:43847 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T02:40:18,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60011, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T02:40:18,500 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43697 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 72b6db343a5d,43847,1732329616195 2024-11-23T02:40:18,502 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43697 {}] master.ServerManager(517): Registering regionserver=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:18,517 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:40:18,518 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46563 2024-11-23T02:40:18,518 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T02:40:18,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T02:40:18,524 DEBUG [RS:0;72b6db343a5d:43847 {}] zookeeper.ZKUtil(111): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/72b6db343a5d,43847,1732329616195 2024-11-23T02:40:18,524 WARN [RS:0;72b6db343a5d:43847 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T02:40:18,524 INFO [RS:0;72b6db343a5d:43847 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T02:40:18,525 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195 2024-11-23T02:40:18,526 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [72b6db343a5d,43847,1732329616195] 2024-11-23T02:40:18,552 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T02:40:18,567 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T02:40:18,575 INFO [RS:0;72b6db343a5d:43847 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T02:40:18,575 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,576 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T02:40:18,582 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T02:40:18,583 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,584 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,584 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,584 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,584 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,585 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,585 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/72b6db343a5d:0, corePoolSize=2, maxPoolSize=2 2024-11-23T02:40:18,585 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,585 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,586 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,586 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,586 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,586 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/72b6db343a5d:0, corePoolSize=1, maxPoolSize=1 2024-11-23T02:40:18,587 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/72b6db343a5d:0, corePoolSize=3, maxPoolSize=3 2024-11-23T02:40:18,587 DEBUG [RS:0;72b6db343a5d:43847 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/72b6db343a5d:0, corePoolSize=3, maxPoolSize=3 2024-11-23T02:40:18,589 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,589 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,589 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,589 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,590 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,590 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43847,1732329616195-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T02:40:18,611 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T02:40:18,613 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43847,1732329616195-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,613 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,613 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.Replication(171): 72b6db343a5d,43847,1732329616195 started 2024-11-23T02:40:18,633 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:18,633 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1482): Serving as 72b6db343a5d,43847,1732329616195, RpcServer on 72b6db343a5d/172.17.0.2:43847, sessionid=0x10142b217ae0001 2024-11-23T02:40:18,634 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T02:40:18,634 DEBUG [RS:0;72b6db343a5d:43847 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 72b6db343a5d,43847,1732329616195 2024-11-23T02:40:18,634 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '72b6db343a5d,43847,1732329616195' 2024-11-23T02:40:18,634 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T02:40:18,635 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T02:40:18,636 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T02:40:18,636 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T02:40:18,637 DEBUG [RS:0;72b6db343a5d:43847 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 72b6db343a5d,43847,1732329616195 2024-11-23T02:40:18,637 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '72b6db343a5d,43847,1732329616195' 2024-11-23T02:40:18,637 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T02:40:18,637 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T02:40:18,638 DEBUG [RS:0;72b6db343a5d:43847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T02:40:18,638 INFO [RS:0;72b6db343a5d:43847 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T02:40:18,638 INFO [RS:0;72b6db343a5d:43847 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T02:40:18,841 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T02:40:18,842 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:40:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741832_1008 (size=32) 2024-11-23T02:40:19,253 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:19,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T02:40:19,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T02:40:19,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:19,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:19,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T02:40:19,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T02:40:19,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:19,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:19,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T02:40:19,266 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T02:40:19,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:19,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:19,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T02:40:19,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T02:40:19,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:19,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:19,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T02:40:19,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740 2024-11-23T02:40:19,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740 2024-11-23T02:40:19,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T02:40:19,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T02:40:19,276 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T02:40:19,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T02:40:19,282 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:40:19,283 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68412097, jitterRate=0.019419685006141663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T02:40:19,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732329619253Initializing all the Stores at 1732329619255 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329619255Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329619256 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329619256Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329619256Cleaning up temporary data from old regions at 1732329619275 (+19 ms)Region opened successfully at 1732329619285 (+10 ms) 2024-11-23T02:40:19,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T02:40:19,285 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T02:40:19,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T02:40:19,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T02:40:19,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T02:40:19,287 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T02:40:19,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732329619285Disabling compacts and flushes for region at 1732329619285Disabling writes for close at 1732329619285Writing region close event to WAL at 1732329619286 (+1 ms)Closed at 1732329619287 (+1 ms) 2024-11-23T02:40:19,290 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T02:40:19,290 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T02:40:19,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T02:40:19,305 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T02:40:19,307 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T02:40:19,458 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:20,459 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:21,460 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:22,460 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:23,461 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:24,462 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:25,462 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:25,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T02:40:25,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T02:40:26,463 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:27,463 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:28,464 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:29,465 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:30,465 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:31,466 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:32,466 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:33,467 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:34,468 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:35,468 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:36,469 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:37,470 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:38,470 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:39,471 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:40,472 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:41,476 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:42,482 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:43,485 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:44,485 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:44,556 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T02:40:45,490 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:46,495 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:47,499 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:48,505 WARN [72b6db343a5d:43697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T02:40:48,654 INFO [RS:0;72b6db343a5d:43847 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T02:40:48,677 INFO [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=72b6db343a5d%2C43847%2C1732329616195, suffix=, logDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195, archiveDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/oldWALs, maxLogs=32 2024-11-23T02:40:48,711 DEBUG [RS:0;72b6db343a5d:43847 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195/72b6db343a5d%2C43847%2C1732329616195.1732329648682, exclude list is [], retry=0 2024-11-23T02:40:48,739 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45173,DS-ebc9641e-4e9b-496b-8a11-6e62876b6f42,DISK] 2024-11-23T02:40:48,844 INFO [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195/72b6db343a5d%2C43847%2C1732329616195.1732329648682 2024-11-23T02:40:48,858 DEBUG [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33259:33259)] 2024-11-23T02:40:49,008 DEBUG [72b6db343a5d:43697 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T02:40:49,057 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:49,066 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 72b6db343a5d,43847,1732329616195, state=OPENING 2024-11-23T02:40:49,131 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T02:40:49,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:49,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:40:49,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T02:40:49,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T02:40:49,223 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T02:40:49,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:40:49,697 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T02:40:49,836 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40089, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T02:40:50,043 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T02:40:50,043 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T02:40:50,044 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T02:40:50,070 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=72b6db343a5d%2C43847%2C1732329616195.meta, suffix=.meta, logDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195, archiveDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/oldWALs, maxLogs=32 2024-11-23T02:40:50,123 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195/72b6db343a5d%2C43847%2C1732329616195.meta.1732329650084.meta, exclude list is [], retry=0 2024-11-23T02:40:50,132 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45173,DS-ebc9641e-4e9b-496b-8a11-6e62876b6f42,DISK] 2024-11-23T02:40:50,297 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195/72b6db343a5d%2C43847%2C1732329616195.meta.1732329650084.meta 2024-11-23T02:40:50,313 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33259:33259)] 2024-11-23T02:40:50,314 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:40:50,316 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T02:40:50,320 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T02:40:50,328 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T02:40:50,334 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T02:40:50,347 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:50,348 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T02:40:50,348 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T02:40:50,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T02:40:50,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T02:40:50,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:50,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:50,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T02:40:50,549 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T02:40:50,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:50,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:50,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T02:40:50,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T02:40:50,554 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:50,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:50,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T02:40:50,557 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T02:40:50,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:50,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T02:40:50,558 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T02:40:50,565 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740 2024-11-23T02:40:50,568 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740 2024-11-23T02:40:50,571 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T02:40:50,572 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T02:40:50,573 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T02:40:50,579 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T02:40:50,582 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64041293, jitterRate=-0.04571036994457245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T02:40:50,582 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T02:40:50,586 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732329650459Writing region info on filesystem at 1732329650459Initializing all the Stores at 1732329650508 (+49 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329650508Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329650510 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329650510Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732329650510Cleaning up temporary data from old regions at 1732329650572 (+62 ms)Running coprocessor post-open hooks at 1732329650582 (+10 ms)Region opened successfully at 1732329650586 (+4 ms) 2024-11-23T02:40:50,618 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732329649586 2024-11-23T02:40:50,668 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:50,668 DEBUG [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T02:40:50,673 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 72b6db343a5d,43847,1732329616195, state=OPEN 2024-11-23T02:40:50,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T02:40:50,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T02:40:50,696 INFO [RS_OPEN_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T02:40:50,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T02:40:50,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T02:40:50,698 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:50,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T02:40:50,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=72b6db343a5d,43847,1732329616195 in 1.4200 sec 2024-11-23T02:40:50,743 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T02:40:50,743 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T02:40:50,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T02:40:50,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 31.4300 sec 2024-11-23T02:40:50,942 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T02:40:50,944 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=72b6db343a5d,43847,1732329616195, seqNum=-1] 2024-11-23T02:40:51,122 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T02:40:51,137 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34237, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T02:40:51,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 32.8990 sec 2024-11-23T02:40:51,230 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732329651230, completionTime=-1 2024-11-23T02:40:51,234 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T02:40:51,234 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T02:40:51,444 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T02:40:51,448 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732329711448 2024-11-23T02:40:51,448 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732329771448 2024-11-23T02:40:51,448 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 213 msec 2024-11-23T02:40:51,460 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43697,1732329615460-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,460 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43697,1732329615460-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,460 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43697,1732329615460-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,470 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-72b6db343a5d:43697, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,486 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,491 DEBUG [master/72b6db343a5d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T02:40:51,521 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,604 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 35.265sec 2024-11-23T02:40:51,606 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T02:40:51,608 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T02:40:51,631 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T02:40:51,632 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T02:40:51,633 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T02:40:51,634 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43697,1732329615460-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T02:40:51,635 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43697,1732329615460-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T02:40:51,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T02:40:51,663 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T02:40:51,663 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T02:40:51,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 72b6db343a5d,43697,-1 for getting cluster id 2024-11-23T02:40:51,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T02:40:51,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '49234b28-a0f4-4e16-8eda-c1b946865dbc' 2024-11-23T02:40:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T02:40:51,744 DEBUG [master/72b6db343a5d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T02:40:51,746 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T02:40:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "49234b28-a0f4-4e16-8eda-c1b946865dbc" 2024-11-23T02:40:51,746 INFO [master/72b6db343a5d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=72b6db343a5d,43697,1732329615460-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T02:40:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53071b09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T02:40:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [72b6db343a5d,43697,-1] 2024-11-23T02:40:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T02:40:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:40:51,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T02:40:51,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T02:40:51,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T02:40:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=72b6db343a5d,43847,1732329616195, seqNum=-1] 2024-11-23T02:40:51,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T02:40:52,003 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T02:40:52,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=72b6db343a5d,43697,1732329615460 2024-11-23T02:40:52,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd93155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T02:40:52,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 72b6db343a5d,43697,-1 for getting cluster id 2024-11-23T02:40:52,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T02:40:52,180 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '49234b28-a0f4-4e16-8eda-c1b946865dbc' 2024-11-23T02:40:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T02:40:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "49234b28-a0f4-4e16-8eda-c1b946865dbc" 2024-11-23T02:40:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217979fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T02:40:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [72b6db343a5d,43697,-1] 2024-11-23T02:40:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T02:40:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:40:52,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T02:40:52,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8afe79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T02:40:52,451 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=923, ProcessCount=11, AvailableMemoryMB=6882 2024-11-23T02:40:52,482 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T02:40:52,547 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(321): The fetched master address is 72b6db343a5d,43697,1732329615460 2024-11-23T02:40:52,561 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@489db9b2 2024-11-23T02:40:52,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T02:40:52,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T02:40:52,700 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:40:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-23T02:40:52,853 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:40:52,857 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:52,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-23T02:40:52,865 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:40:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T02:40:53,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741835_1011 (size=428) 2024-11-23T02:40:53,085 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e6f7e4ac90f40f9a72bea324c4ac070, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:40:53,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741836_1012 (size=79) 2024-11-23T02:40:53,319 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:53,319 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4e6f7e4ac90f40f9a72bea324c4ac070, disabling compactions & flushes 2024-11-23T02:40:53,320 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:53,320 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:53,320 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. after waiting 0 ms 2024-11-23T02:40:53,320 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:53,320 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:53,320 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e6f7e4ac90f40f9a72bea324c4ac070: Waiting for close lock at 1732329653319Disabling compacts and flushes for region at 1732329653319Disabling writes for close at 1732329653320 (+1 ms)Writing region close event to WAL at 1732329653320Closed at 1732329653320 2024-11-23T02:40:53,325 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:40:53,332 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732329653325"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329653325"}]},"ts":"1732329653325"} 2024-11-23T02:40:53,345 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:40:53,351 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:40:53,355 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329653351"}]},"ts":"1732329653351"} 2024-11-23T02:40:53,391 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-23T02:40:53,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e6f7e4ac90f40f9a72bea324c4ac070, ASSIGN}] 2024-11-23T02:40:53,474 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e6f7e4ac90f40f9a72bea324c4ac070, ASSIGN 2024-11-23T02:40:53,477 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e6f7e4ac90f40f9a72bea324c4ac070, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:40:53,629 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e6f7e4ac90f40f9a72bea324c4ac070, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:53,635 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e6f7e4ac90f40f9a72bea324c4ac070, ASSIGN because future has completed 2024-11-23T02:40:53,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6f7e4ac90f40f9a72bea324c4ac070, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:40:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T02:40:53,991 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:53,992 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e6f7e4ac90f40f9a72bea324c4ac070, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:40:53,992 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:53,993 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:53,993 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:53,993 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,083 INFO [StoreOpener-4e6f7e4ac90f40f9a72bea324c4ac070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,105 INFO [StoreOpener-4e6f7e4ac90f40f9a72bea324c4ac070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e6f7e4ac90f40f9a72bea324c4ac070 columnFamilyName testFamily 2024-11-23T02:40:54,105 DEBUG [StoreOpener-4e6f7e4ac90f40f9a72bea324c4ac070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:54,108 INFO [StoreOpener-4e6f7e4ac90f40f9a72bea324c4ac070-1 {}] regionserver.HStore(327): Store=4e6f7e4ac90f40f9a72bea324c4ac070/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:40:54,108 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,111 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,112 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,114 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,114 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,118 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,183 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:40:54,184 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4e6f7e4ac90f40f9a72bea324c4ac070; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72100002, jitterRate=0.0743737518787384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:40:54,184 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:40:54,186 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4e6f7e4ac90f40f9a72bea324c4ac070: Running coprocessor pre-open hook at 1732329653993Writing region info on filesystem at 1732329653993Initializing all the Stores at 1732329654058 (+65 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329654058Cleaning up temporary data from old regions at 1732329654114 (+56 ms)Running coprocessor post-open hooks at 1732329654184 (+70 ms)Region opened successfully at 1732329654186 (+2 ms) 2024-11-23T02:40:54,191 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070., pid=6, masterSystemTime=1732329653892 2024-11-23T02:40:54,229 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:54,229 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:40:54,231 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e6f7e4ac90f40f9a72bea324c4ac070, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:54,241 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e6f7e4ac90f40f9a72bea324c4ac070, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:40:54,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T02:40:54,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e6f7e4ac90f40f9a72bea324c4ac070, server=72b6db343a5d,43847,1732329616195 in 570 msec 2024-11-23T02:40:54,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T02:40:54,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e6f7e4ac90f40f9a72bea324c4ac070, ASSIGN in 928 msec 2024-11-23T02:40:54,376 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:40:54,377 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329654376"}]},"ts":"1732329654376"} 2024-11-23T02:40:54,381 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-23T02:40:54,409 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:40:54,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.7050 sec 2024-11-23T02:40:54,547 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-23T02:40:54,557 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T02:40:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T02:40:55,939 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-23T02:40:55,939 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-23T02:40:55,941 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:40:55,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-23T02:40:55,981 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:40:55,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-23T02:40:55,986 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T02:40:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=72b6db343a5d,43847,1732329616195, seqNum=-1] 2024-11-23T02:40:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T02:40:56,011 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T02:40:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:40:56,129 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:40:56,192 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:40:56,204 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-23T02:40:56,210 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-23T02:40:56,260 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-23T02:40:56,267 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-23T02:40:56,281 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2145723291229224952 2024-11-23T02:40:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329659277, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:40:56,378 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T02:40:56,378 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T02:40:56,380 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T02:40:56,380 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T02:40:56,380 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-23T02:40:56,381 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T02:40:56,561 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-23T02:40:56,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-23T02:40:56,575 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:40:56,634 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:40:56,635 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2145723291229224950 2024-11-23T02:40:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329659634, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:40:56,690 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:40:56,691 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-2145723291229224949 2024-11-23T02:40:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329659690, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:40:56,729 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:40:56,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-2145723291229224948 2024-11-23T02:40:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329659738, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:40:56,793 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:40:56,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 app//org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$startScan$1(AsyncClientScanner.java:226) app//org.apache.hadoop.hbase.client.AsyncClientScanner$$Lambda$769/0x00007fbf70b90ec8.accept(Unknown Source) app//org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) app//org.apache.hadoop.hbase.util.FutureUtils$$Lambda$432/0x00007fbf709e3760.accept(Unknown Source) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) java.base@17.0.11/java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) app//org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) app//org.apache.hadoop.hbase.client.AsyncClientScanner.startScan(AsyncClientScanner.java:199) app//org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$openScanner$2(AsyncClientScanner.java:268) app//org.apache.hadoop.hbase.client.AsyncClientScanner$$Lambda$767/0x00007fbf70b8dda0.accept(Unknown Source) app//org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) app//org.apache.hadoop.hbase.util.FutureUtils$$Lambda$432/0x00007fbf709e3760.accept(Unknown Source) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) java.base@17.0.11/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) java.base@17.0.11/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) java.base@17.0.11/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) app//org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:92) app//org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller$$Lambda$745/0x00007fbf70b761c0.accept(Unknown Source) app//org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) app//org.apache.hadoop.hbase.util.FutureUtils$$Lambda$432/0x00007fbf709e3760.accept(Unknown Source) java.base@17.0.11/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) java.base@17.0.11/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) java.base@17.0.11/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) java.base@17.0.11/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) app//org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:187) app//org.apache.hadoop.hbase.client.AsyncClientScanner$$Lambda$760/0x00007fbf70b87dd0.run(Unknown Source) app//org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) app//org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:400) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) app//org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) app//org.apache.hadoop.hbase.ipc.Call.setResponse(Call.java:149) app//org.apache.hadoop.hbase.ipc.RpcConnection.finishCall(RpcConnection.java:396) app//org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:461) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1291806754_22 at /127.0.0.1:41594 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1298 (was 923) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6830 (was 6882) 2024-11-23T02:40:56,919 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1298, ProcessCount=11, AvailableMemoryMB=6830 2024-11-23T02:40:56,924 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:40:56,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-23T02:40:56,971 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:40:56,972 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:56,973 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-23T02:40:56,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:40:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T02:40:57,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741837_1013 (size=414) 2024-11-23T02:40:57,541 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4d4e3f390252890bbd33ce119c8c6eb3, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:40:57,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741838_1014 (size=65) 2024-11-23T02:40:57,761 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:57,761 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 4d4e3f390252890bbd33ce119c8c6eb3, disabling compactions & flushes 2024-11-23T02:40:57,761 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:57,761 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:57,761 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. after waiting 0 ms 2024-11-23T02:40:57,761 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:57,761 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:57,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4d4e3f390252890bbd33ce119c8c6eb3: Waiting for close lock at 1732329657761Disabling compacts and flushes for region at 1732329657761Disabling writes for close at 1732329657761Writing region close event to WAL at 1732329657761Closed at 1732329657761 2024-11-23T02:40:57,764 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:40:57,765 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732329657765"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329657765"}]},"ts":"1732329657765"} 2024-11-23T02:40:57,771 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:40:57,773 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:40:57,774 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329657773"}]},"ts":"1732329657773"} 2024-11-23T02:40:57,777 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-23T02:40:57,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4d4e3f390252890bbd33ce119c8c6eb3, ASSIGN}] 2024-11-23T02:40:57,783 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4d4e3f390252890bbd33ce119c8c6eb3, ASSIGN 2024-11-23T02:40:57,784 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4d4e3f390252890bbd33ce119c8c6eb3, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:40:57,936 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4d4e3f390252890bbd33ce119c8c6eb3, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:57,943 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4d4e3f390252890bbd33ce119c8c6eb3, ASSIGN because future has completed 2024-11-23T02:40:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T02:40:58,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d4e3f390252890bbd33ce119c8c6eb3, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:40:58,276 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:58,277 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d4e3f390252890bbd33ce119c8c6eb3, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:40:58,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:40:58,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,295 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,338 INFO [StoreOpener-4d4e3f390252890bbd33ce119c8c6eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,472 INFO [StoreOpener-4d4e3f390252890bbd33ce119c8c6eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d4e3f390252890bbd33ce119c8c6eb3 columnFamilyName testFamily 2024-11-23T02:40:58,472 DEBUG [StoreOpener-4d4e3f390252890bbd33ce119c8c6eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:40:58,506 INFO [StoreOpener-4d4e3f390252890bbd33ce119c8c6eb3-1 {}] regionserver.HStore(327): Store=4d4e3f390252890bbd33ce119c8c6eb3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:40:58,506 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,508 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,513 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,513 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,545 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:40:58,548 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 4d4e3f390252890bbd33ce119c8c6eb3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64131481, jitterRate=-0.044366464018821716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:40:58,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:40:58,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 4d4e3f390252890bbd33ce119c8c6eb3: Running coprocessor pre-open hook at 1732329658295Writing region info on filesystem at 1732329658295Initializing all the Stores at 1732329658333 (+38 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329658333Cleaning up temporary data from old regions at 1732329658513 (+180 ms)Running coprocessor post-open hooks at 1732329658548 (+35 ms)Region opened successfully at 1732329658549 (+1 ms) 2024-11-23T02:40:58,591 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., pid=9, masterSystemTime=1732329658209 2024-11-23T02:40:58,598 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4d4e3f390252890bbd33ce119c8c6eb3, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:40:58,603 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d4e3f390252890bbd33ce119c8c6eb3, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:40:58,604 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:58,604 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:40:58,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T02:40:58,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d4e3f390252890bbd33ce119c8c6eb3, server=72b6db343a5d,43847,1732329616195 in 634 msec 2024-11-23T02:40:58,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T02:40:58,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4d4e3f390252890bbd33ce119c8c6eb3, ASSIGN in 896 msec 2024-11-23T02:40:58,681 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:40:58,681 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329658681"}]},"ts":"1732329658681"} 2024-11-23T02:40:58,685 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-23T02:40:58,687 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:40:58,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.7630 sec 2024-11-23T02:41:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T02:41:00,020 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-23T02:41:00,020 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-23T02:41:00,021 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:00,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-23T02:41:00,034 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:00,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-23T02:41:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:00,062 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:00,062 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-23T02:41:00,068 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T02:41:00,071 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-23T02:41:00,565 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-23T02:41:03,795 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:05,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-23T02:41:05,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:06,086 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., hostname=72b6db343a5d,43847,1732329616195, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=72b6db343a5d:43847 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6014ms, rpcTimeout=5999ms 2024-11-23T02:41:06,086 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6014ms, rpcTimeout=5999ms 2024-11-23T02:41:06,086 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 from cache 2024-11-23T02:41:06,096 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1552 (was 1298) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6829 (was 6830) 2024-11-23T02:41:06,104 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1552, ProcessCount=11, AvailableMemoryMB=6829 2024-11-23T02:41:06,108 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:06,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-23T02:41:06,112 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:06,112 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:06,112 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-23T02:41:06,113 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-23T02:41:06,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741839_1015 (size=412) 2024-11-23T02:41:06,526 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ddd12496f82516631f13ec470bb219c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:06,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741840_1016 (size=63) 2024-11-23T02:41:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-23T02:41:06,935 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:06,935 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2ddd12496f82516631f13ec470bb219c, disabling compactions & flushes 2024-11-23T02:41:06,936 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:06,936 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:06,936 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. after waiting 0 ms 2024-11-23T02:41:06,936 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:06,936 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:06,936 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ddd12496f82516631f13ec470bb219c: Waiting for close lock at 1732329666935Disabling compacts and flushes for region at 1732329666935Disabling writes for close at 1732329666936 (+1 ms)Writing region close event to WAL at 1732329666936Closed at 1732329666936 2024-11-23T02:41:06,938 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:06,939 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732329666938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329666938"}]},"ts":"1732329666938"} 2024-11-23T02:41:06,942 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:06,944 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:06,944 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329666944"}]},"ts":"1732329666944"} 2024-11-23T02:41:06,948 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-23T02:41:06,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=2ddd12496f82516631f13ec470bb219c, ASSIGN}] 2024-11-23T02:41:06,950 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=2ddd12496f82516631f13ec470bb219c, ASSIGN 2024-11-23T02:41:06,951 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=2ddd12496f82516631f13ec470bb219c, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:07,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=2ddd12496f82516631f13ec470bb219c, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:07,106 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=2ddd12496f82516631f13ec470bb219c, ASSIGN because future has completed 2024-11-23T02:41:07,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ddd12496f82516631f13ec470bb219c, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-23T02:41:07,266 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:07,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ddd12496f82516631f13ec470bb219c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:07,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:07,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,269 INFO [StoreOpener-2ddd12496f82516631f13ec470bb219c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,271 INFO [StoreOpener-2ddd12496f82516631f13ec470bb219c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ddd12496f82516631f13ec470bb219c columnFamilyName testFamily 2024-11-23T02:41:07,272 DEBUG [StoreOpener-2ddd12496f82516631f13ec470bb219c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:07,272 INFO [StoreOpener-2ddd12496f82516631f13ec470bb219c-1 {}] regionserver.HStore(327): Store=2ddd12496f82516631f13ec470bb219c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:07,272 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,275 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,275 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,278 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,281 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:07,282 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 2ddd12496f82516631f13ec470bb219c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66814793, jitterRate=-0.004381999373435974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:07,282 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:07,283 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 2ddd12496f82516631f13ec470bb219c: Running coprocessor pre-open hook at 1732329667267Writing region info on filesystem at 1732329667267Initializing all the Stores at 1732329667269 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329667269Cleaning up temporary data from old regions at 1732329667275 (+6 ms)Running coprocessor post-open hooks at 1732329667282 (+7 ms)Region opened successfully at 1732329667283 (+1 ms) 2024-11-23T02:41:07,285 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c., pid=12, masterSystemTime=1732329667261 2024-11-23T02:41:07,288 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:07,288 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:41:07,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=2ddd12496f82516631f13ec470bb219c, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:07,292 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ddd12496f82516631f13ec470bb219c, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:07,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T02:41:07,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ddd12496f82516631f13ec470bb219c, server=72b6db343a5d,43847,1732329616195 in 187 msec 2024-11-23T02:41:07,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-23T02:41:07,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=2ddd12496f82516631f13ec470bb219c, ASSIGN in 349 msec 2024-11-23T02:41:07,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:07,301 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329667301"}]},"ts":"1732329667301"} 2024-11-23T02:41:07,304 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-23T02:41:07,305 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:07,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.1970 sec 2024-11-23T02:41:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-23T02:41:09,159 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-23T02:41:09,159 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-23T02:41:09,159 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:09,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-23T02:41:09,165 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:09,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-23T02:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:09,180 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:09,180 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-23T02:41:09,183 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T02:41:09,185 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T02:41:09,186 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T02:41:09,187 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-23T02:41:12,769 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:12,787 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-23T02:41:14,556 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T02:41:15,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224939' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T02:41:15,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60464 deadline: 1732329681192, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224939' does not exist 2024-11-23T02:41:15,198 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 72b6db343a5d,43847,1732329616195 for closing scanner id = -2145723291229224939 for 2ddd12496f82516631f13ec470bb219c of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224939' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T02:41:15,201 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 223), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1313 (was 1552), ProcessCount=11 (was 11), AvailableMemoryMB=8181 (was 6829) - AvailableMemoryMB LEAK? - 2024-11-23T02:41:15,211 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1313, ProcessCount=11, AvailableMemoryMB=8180 2024-11-23T02:41:15,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-23T02:41:15,219 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:15,219 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:15,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-23T02:41:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T02:41:15,221 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:15,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741841_1017 (size=423) 2024-11-23T02:41:15,233 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1cf4cc61d248d947709cd85539a97fa8, NAME => 'testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:15,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741842_1018 (size=74) 2024-11-23T02:41:15,244 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:15,245 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 1cf4cc61d248d947709cd85539a97fa8, disabling compactions & flushes 2024-11-23T02:41:15,245 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,245 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,245 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. after waiting 0 ms 2024-11-23T02:41:15,245 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,245 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,245 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1cf4cc61d248d947709cd85539a97fa8: Waiting for close lock at 1732329675245Disabling compacts and flushes for region at 1732329675245Disabling writes for close at 1732329675245Writing region close event to WAL at 1732329675245Closed at 1732329675245 2024-11-23T02:41:15,247 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:15,248 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732329675247"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329675247"}]},"ts":"1732329675247"} 2024-11-23T02:41:15,251 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:15,253 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:15,253 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329675253"}]},"ts":"1732329675253"} 2024-11-23T02:41:15,256 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-23T02:41:15,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1cf4cc61d248d947709cd85539a97fa8, ASSIGN}] 2024-11-23T02:41:15,258 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1cf4cc61d248d947709cd85539a97fa8, ASSIGN 2024-11-23T02:41:15,260 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1cf4cc61d248d947709cd85539a97fa8, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:15,411 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1cf4cc61d248d947709cd85539a97fa8, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:15,416 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1cf4cc61d248d947709cd85539a97fa8, ASSIGN because future has completed 2024-11-23T02:41:15,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf4cc61d248d947709cd85539a97fa8, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:15,577 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,578 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cf4cc61d248d947709cd85539a97fa8, NAME => 'testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:15,578 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,578 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:15,578 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,578 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,580 INFO [StoreOpener-1cf4cc61d248d947709cd85539a97fa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,583 INFO [StoreOpener-1cf4cc61d248d947709cd85539a97fa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cf4cc61d248d947709cd85539a97fa8 columnFamilyName testFamily 2024-11-23T02:41:15,584 DEBUG [StoreOpener-1cf4cc61d248d947709cd85539a97fa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:15,584 INFO [StoreOpener-1cf4cc61d248d947709cd85539a97fa8-1 {}] regionserver.HStore(327): Store=1cf4cc61d248d947709cd85539a97fa8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:15,585 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,585 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,586 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,587 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,587 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,589 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,593 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:15,594 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 1cf4cc61d248d947709cd85539a97fa8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62887099, jitterRate=-0.06290920078754425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:15,594 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:15,595 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 1cf4cc61d248d947709cd85539a97fa8: Running coprocessor pre-open hook at 1732329675578Writing region info on filesystem at 1732329675578Initializing all the Stores at 1732329675580 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329675580Cleaning up temporary data from old regions at 1732329675587 (+7 ms)Running coprocessor post-open hooks at 1732329675594 (+7 ms)Region opened successfully at 1732329675595 (+1 ms) 2024-11-23T02:41:15,597 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8., pid=15, masterSystemTime=1732329675571 2024-11-23T02:41:15,601 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,601 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:41:15,602 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1cf4cc61d248d947709cd85539a97fa8, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:15,606 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cf4cc61d248d947709cd85539a97fa8, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:15,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-23T02:41:15,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 1cf4cc61d248d947709cd85539a97fa8, server=72b6db343a5d,43847,1732329616195 in 192 msec 2024-11-23T02:41:15,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T02:41:15,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1cf4cc61d248d947709cd85539a97fa8, ASSIGN in 355 msec 2024-11-23T02:41:15,616 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:15,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329675616"}]},"ts":"1732329675616"} 2024-11-23T02:41:15,620 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-23T02:41:15,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:15,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 407 msec 2024-11-23T02:41:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-23T02:41:15,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-23T02:41:15,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:15,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-23T02:41:15,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T02:41:16,228 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-23T02:41:16,228 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-23T02:41:16,228 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:16,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-23T02:41:16,233 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:16,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-23T02:41:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:16,250 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:16,250 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-23T02:41:16,254 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-23T02:41:16,257 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-23T02:41:16,259 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-23T02:41:16,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2145723291229224935 2024-11-23T02:41:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329679258, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:41:16,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-23T02:41:16,265 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-23T02:41:16,266 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:41:16,267 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:41:16,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2145723291229224933 2024-11-23T02:41:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329679267, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:41:16,271 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:41:16,272 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-2145723291229224932 2024-11-23T02:41:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329679270, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:41:16,275 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:41:16,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-2145723291229224931 2024-11-23T02:41:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60464 deadline: 1732329679274, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-23T02:41:16,279 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-23T02:41:16,290 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1291806754_22 at /127.0.0.1:52112 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1313 (was 1313), ProcessCount=11 (was 11), AvailableMemoryMB=8088 (was 8180) 2024-11-23T02:41:16,298 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1313, ProcessCount=11, AvailableMemoryMB=8086 2024-11-23T02:41:16,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-23T02:41:16,305 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:16,305 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:16,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-23T02:41:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-23T02:41:16,308 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:16,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741843_1019 (size=418) 2024-11-23T02:41:16,722 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64f9f4c22f3b6beb764abd436719011a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:16,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741844_1020 (size=69) 2024-11-23T02:41:17,134 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:17,134 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 64f9f4c22f3b6beb764abd436719011a, disabling compactions & flushes 2024-11-23T02:41:17,134 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,134 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,134 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. after waiting 0 ms 2024-11-23T02:41:17,134 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,134 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,134 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64f9f4c22f3b6beb764abd436719011a: Waiting for close lock at 1732329677134Disabling compacts and flushes for region at 1732329677134Disabling writes for close at 1732329677134Writing region close event to WAL at 1732329677134Closed at 1732329677134 2024-11-23T02:41:17,137 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:17,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732329677137"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329677137"}]},"ts":"1732329677137"} 2024-11-23T02:41:17,141 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:17,143 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:17,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329677143"}]},"ts":"1732329677143"} 2024-11-23T02:41:17,147 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-23T02:41:17,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=64f9f4c22f3b6beb764abd436719011a, ASSIGN}] 2024-11-23T02:41:17,149 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=64f9f4c22f3b6beb764abd436719011a, ASSIGN 2024-11-23T02:41:17,150 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=64f9f4c22f3b6beb764abd436719011a, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:17,301 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=64f9f4c22f3b6beb764abd436719011a, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:17,304 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=64f9f4c22f3b6beb764abd436719011a, ASSIGN because future has completed 2024-11-23T02:41:17,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64f9f4c22f3b6beb764abd436719011a, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-23T02:41:17,464 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,465 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 64f9f4c22f3b6beb764abd436719011a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:17,465 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,466 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:17,466 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,466 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,468 INFO [StoreOpener-64f9f4c22f3b6beb764abd436719011a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,471 INFO [StoreOpener-64f9f4c22f3b6beb764abd436719011a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64f9f4c22f3b6beb764abd436719011a columnFamilyName testFamily 2024-11-23T02:41:17,471 DEBUG [StoreOpener-64f9f4c22f3b6beb764abd436719011a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:17,471 INFO [StoreOpener-64f9f4c22f3b6beb764abd436719011a-1 {}] regionserver.HStore(327): Store=64f9f4c22f3b6beb764abd436719011a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:17,471 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,473 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,474 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,474 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,475 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,477 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,485 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:17,486 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 64f9f4c22f3b6beb764abd436719011a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75467471, jitterRate=0.12455295026302338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:17,486 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:17,487 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 64f9f4c22f3b6beb764abd436719011a: Running coprocessor pre-open hook at 1732329677466Writing region info on filesystem at 1732329677466Initializing all the Stores at 1732329677468 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329677468Cleaning up temporary data from old regions at 1732329677475 (+7 ms)Running coprocessor post-open hooks at 1732329677486 (+11 ms)Region opened successfully at 1732329677487 (+1 ms) 2024-11-23T02:41:17,488 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., pid=18, masterSystemTime=1732329677458 2024-11-23T02:41:17,492 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,492 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:41:17,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=64f9f4c22f3b6beb764abd436719011a, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:17,496 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64f9f4c22f3b6beb764abd436719011a, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:17,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-23T02:41:17,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 64f9f4c22f3b6beb764abd436719011a, server=72b6db343a5d,43847,1732329616195 in 193 msec 2024-11-23T02:41:17,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-23T02:41:17,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=64f9f4c22f3b6beb764abd436719011a, ASSIGN in 355 msec 2024-11-23T02:41:17,507 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:17,508 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329677507"}]},"ts":"1732329677507"} 2024-11-23T02:41:17,511 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-23T02:41:17,513 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:17,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2120 sec 2024-11-23T02:41:18,788 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-23T02:41:18,789 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-23T02:41:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-23T02:41:19,329 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-23T02:41:19,329 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-23T02:41:19,329 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:19,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-23T02:41:19,334 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:19,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-23T02:41:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:19,352 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:19,352 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-23T02:41:19,356 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T02:41:19,357 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T02:41:20,359 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=72b6db343a5d:43847 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-23T02:41:20,360 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-23T02:41:20,360 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 from cache 2024-11-23T02:41:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-23T02:41:21,080 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:21,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T02:41:22,359 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=72b6db343a5d:43847 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=978ms, rpcTimeout=975ms 2024-11-23T02:41:22,359 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=978ms, rpcTimeout=975ms 2024-11-23T02:41:22,359 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 from cache 2024-11-23T02:41:22,369 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1232 (was 1313), ProcessCount=11 (was 11), AvailableMemoryMB=7712 (was 8086) 2024-11-23T02:41:22,377 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=1232, ProcessCount=11, AvailableMemoryMB=7712 2024-11-23T02:41:22,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-23T02:41:22,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:22,383 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:22,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-23T02:41:22,384 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T02:41:22,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741845_1021 (size=416) 2024-11-23T02:41:22,795 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6cf9b9df16853f5abc52ca3755323dd, NAME => 'testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:22,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741846_1022 (size=67) 2024-11-23T02:41:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-23T02:41:23,204 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:23,204 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing e6cf9b9df16853f5abc52ca3755323dd, disabling compactions & flushes 2024-11-23T02:41:23,205 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,205 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,205 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. after waiting 0 ms 2024-11-23T02:41:23,205 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,205 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,205 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6cf9b9df16853f5abc52ca3755323dd: Waiting for close lock at 1732329683204Disabling compacts and flushes for region at 1732329683204Disabling writes for close at 1732329683205 (+1 ms)Writing region close event to WAL at 1732329683205Closed at 1732329683205 2024-11-23T02:41:23,207 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:23,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732329683207"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329683207"}]},"ts":"1732329683207"} 2024-11-23T02:41:23,210 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:23,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:23,212 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329683212"}]},"ts":"1732329683212"} 2024-11-23T02:41:23,214 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-23T02:41:23,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e6cf9b9df16853f5abc52ca3755323dd, ASSIGN}] 2024-11-23T02:41:23,216 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e6cf9b9df16853f5abc52ca3755323dd, ASSIGN 2024-11-23T02:41:23,217 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e6cf9b9df16853f5abc52ca3755323dd, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:23,368 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e6cf9b9df16853f5abc52ca3755323dd, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:23,371 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e6cf9b9df16853f5abc52ca3755323dd, ASSIGN because future has completed 2024-11-23T02:41:23,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6cf9b9df16853f5abc52ca3755323dd, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T02:41:23,531 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,531 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => e6cf9b9df16853f5abc52ca3755323dd, NAME => 'testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:23,532 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,532 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:23,532 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,532 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,534 INFO [StoreOpener-e6cf9b9df16853f5abc52ca3755323dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,537 INFO [StoreOpener-e6cf9b9df16853f5abc52ca3755323dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6cf9b9df16853f5abc52ca3755323dd columnFamilyName testFamily 2024-11-23T02:41:23,537 DEBUG [StoreOpener-e6cf9b9df16853f5abc52ca3755323dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:23,537 INFO [StoreOpener-e6cf9b9df16853f5abc52ca3755323dd-1 {}] regionserver.HStore(327): Store=e6cf9b9df16853f5abc52ca3755323dd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:23,537 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,538 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,539 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,540 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,540 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,542 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,545 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:23,546 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened e6cf9b9df16853f5abc52ca3755323dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66073497, jitterRate=-0.015428170561790466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:23,546 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:23,547 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for e6cf9b9df16853f5abc52ca3755323dd: Running coprocessor pre-open hook at 1732329683533Writing region info on filesystem at 1732329683533Initializing all the Stores at 1732329683534 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329683534Cleaning up temporary data from old regions at 1732329683540 (+6 ms)Running coprocessor post-open hooks at 1732329683546 (+6 ms)Region opened successfully at 1732329683547 (+1 ms) 2024-11-23T02:41:23,548 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd., pid=21, masterSystemTime=1732329683525 2024-11-23T02:41:23,551 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,551 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:41:23,552 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e6cf9b9df16853f5abc52ca3755323dd, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:23,554 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6cf9b9df16853f5abc52ca3755323dd, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:23,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-23T02:41:23,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e6cf9b9df16853f5abc52ca3755323dd, server=72b6db343a5d,43847,1732329616195 in 184 msec 2024-11-23T02:41:23,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-23T02:41:23,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e6cf9b9df16853f5abc52ca3755323dd, ASSIGN in 344 msec 2024-11-23T02:41:23,562 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:23,562 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329683562"}]},"ts":"1732329683562"} 2024-11-23T02:41:23,564 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-23T02:41:23,565 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:23,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1850 sec 2024-11-23T02:41:24,790 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-23T02:41:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T02:41:25,428 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-23T02:41:25,428 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-23T02:41:25,428 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:25,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-23T02:41:25,432 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:25,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-23T02:41:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:25,446 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:25,446 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-23T02:41:25,449 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T02:41:25,451 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T02:41:25,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T02:41:25,453 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-23T02:41:25,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-23T02:41:25,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:25,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-23T02:41:25,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:28,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224920' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T02:41:28,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60464 deadline: 1732329689462, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224920' does not exist 2024-11-23T02:41:28,463 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 72b6db343a5d,43847,1732329616195 for closing scanner id = -2145723291229224920 for e6cf9b9df16853f5abc52ca3755323dd of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224920' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T02:41:28,469 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1065 (was 1232), ProcessCount=11 (was 11), AvailableMemoryMB=7606 (was 7712) 2024-11-23T02:41:28,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1065, ProcessCount=11, AvailableMemoryMB=7606 2024-11-23T02:41:28,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-23T02:41:28,486 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:28,486 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:28,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-23T02:41:28,488 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-23T02:41:28,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741847_1023 (size=411) 2024-11-23T02:41:28,901 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 423e69121a389b50524067c0923919d5, NAME => 'testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:28,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741848_1024 (size=62) 2024-11-23T02:41:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-23T02:41:29,034 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:29,310 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:29,311 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 423e69121a389b50524067c0923919d5, disabling compactions & flushes 2024-11-23T02:41:29,311 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,311 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,311 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. after waiting 0 ms 2024-11-23T02:41:29,311 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,311 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,311 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 423e69121a389b50524067c0923919d5: Waiting for close lock at 1732329689310Disabling compacts and flushes for region at 1732329689310Disabling writes for close at 1732329689311 (+1 ms)Writing region close event to WAL at 1732329689311Closed at 1732329689311 2024-11-23T02:41:29,313 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:29,313 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732329689313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329689313"}]},"ts":"1732329689313"} 2024-11-23T02:41:29,316 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:29,317 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:29,318 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329689318"}]},"ts":"1732329689318"} 2024-11-23T02:41:29,320 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-23T02:41:29,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=423e69121a389b50524067c0923919d5, ASSIGN}] 2024-11-23T02:41:29,323 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=423e69121a389b50524067c0923919d5, ASSIGN 2024-11-23T02:41:29,324 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=423e69121a389b50524067c0923919d5, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:29,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=423e69121a389b50524067c0923919d5, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:29,478 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=423e69121a389b50524067c0923919d5, ASSIGN because future has completed 2024-11-23T02:41:29,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 423e69121a389b50524067c0923919d5, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-23T02:41:29,636 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,636 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 423e69121a389b50524067c0923919d5, NAME => 'testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:29,637 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,637 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:29,637 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,637 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,639 INFO [StoreOpener-423e69121a389b50524067c0923919d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,641 INFO [StoreOpener-423e69121a389b50524067c0923919d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 423e69121a389b50524067c0923919d5 columnFamilyName testFamily 2024-11-23T02:41:29,641 DEBUG [StoreOpener-423e69121a389b50524067c0923919d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:29,641 INFO [StoreOpener-423e69121a389b50524067c0923919d5-1 {}] regionserver.HStore(327): Store=423e69121a389b50524067c0923919d5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:29,641 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,643 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,643 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,644 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,644 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,646 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,649 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:29,650 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 423e69121a389b50524067c0923919d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73733303, jitterRate=0.09871183335781097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:29,650 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:29,650 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 423e69121a389b50524067c0923919d5: Running coprocessor pre-open hook at 1732329689637Writing region info on filesystem at 1732329689637Initializing all the Stores at 1732329689638 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329689638Cleaning up temporary data from old regions at 1732329689644 (+6 ms)Running coprocessor post-open hooks at 1732329689650 (+6 ms)Region opened successfully at 1732329689650 2024-11-23T02:41:29,652 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5., pid=24, masterSystemTime=1732329689631 2024-11-23T02:41:29,654 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,654 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:41:29,655 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=423e69121a389b50524067c0923919d5, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:29,657 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 423e69121a389b50524067c0923919d5, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:29,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-23T02:41:29,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 423e69121a389b50524067c0923919d5, server=72b6db343a5d,43847,1732329616195 in 180 msec 2024-11-23T02:41:29,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-23T02:41:29,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=423e69121a389b50524067c0923919d5, ASSIGN in 340 msec 2024-11-23T02:41:29,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:29,665 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329689665"}]},"ts":"1732329689665"} 2024-11-23T02:41:29,667 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-23T02:41:29,668 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:29,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1850 sec 2024-11-23T02:41:30,791 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-23T02:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-23T02:41:31,518 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-23T02:41:31,518 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-23T02:41:31,518 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:31,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-23T02:41:31,523 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:31,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-23T02:41:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:31,537 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:31,537 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-23T02:41:31,540 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T02:41:31,542 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T02:41:31,543 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T02:41:31,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-23T02:41:32,245 INFO [master/72b6db343a5d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T02:41:32,245 INFO [master/72b6db343a5d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T02:41:34,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43847 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224916' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T02:41:34,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60464 deadline: 1732329695552, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224916' does not exist 2024-11-23T02:41:34,553 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 72b6db343a5d,43847,1732329616195 for closing scanner id = -2145723291229224916 for 423e69121a389b50524067c0923919d5 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224916' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T02:41:34,559 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=979 (was 1065), ProcessCount=11 (was 11), AvailableMemoryMB=7553 (was 7606) 2024-11-23T02:41:34,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=979, ProcessCount=11, AvailableMemoryMB=7552 2024-11-23T02:41:34,572 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:34,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-23T02:41:34,575 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:34,575 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:34,575 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-23T02:41:34,576 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T02:41:34,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741849_1025 (size=419) 2024-11-23T02:41:34,989 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cddeb9d84308524ecd65b2ecb2ba925b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:34,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741850_1026 (size=70) 2024-11-23T02:41:35,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-23T02:41:35,138 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing cddeb9d84308524ecd65b2ecb2ba925b, disabling compactions & flushes 2024-11-23T02:41:35,397 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. after waiting 0 ms 2024-11-23T02:41:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,397 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,398 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cddeb9d84308524ecd65b2ecb2ba925b: Waiting for close lock at 1732329695397Disabling compacts and flushes for region at 1732329695397Disabling writes for close at 1732329695397Writing region close event to WAL at 1732329695397Closed at 1732329695397 2024-11-23T02:41:35,399 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:35,400 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732329695399"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329695399"}]},"ts":"1732329695399"} 2024-11-23T02:41:35,403 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:35,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:35,404 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329695404"}]},"ts":"1732329695404"} 2024-11-23T02:41:35,406 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-23T02:41:35,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cddeb9d84308524ecd65b2ecb2ba925b, ASSIGN}] 2024-11-23T02:41:35,408 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cddeb9d84308524ecd65b2ecb2ba925b, ASSIGN 2024-11-23T02:41:35,409 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cddeb9d84308524ecd65b2ecb2ba925b, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:35,560 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=cddeb9d84308524ecd65b2ecb2ba925b, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:35,563 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cddeb9d84308524ecd65b2ecb2ba925b, ASSIGN because future has completed 2024-11-23T02:41:35,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure cddeb9d84308524ecd65b2ecb2ba925b, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T02:41:35,722 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => cddeb9d84308524ecd65b2ecb2ba925b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:35,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:35,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,725 INFO [StoreOpener-cddeb9d84308524ecd65b2ecb2ba925b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,727 INFO [StoreOpener-cddeb9d84308524ecd65b2ecb2ba925b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddeb9d84308524ecd65b2ecb2ba925b columnFamilyName testFamily 2024-11-23T02:41:35,727 DEBUG [StoreOpener-cddeb9d84308524ecd65b2ecb2ba925b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:35,728 INFO [StoreOpener-cddeb9d84308524ecd65b2ecb2ba925b-1 {}] regionserver.HStore(327): Store=cddeb9d84308524ecd65b2ecb2ba925b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:35,728 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,732 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,735 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:35,736 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened cddeb9d84308524ecd65b2ecb2ba925b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61670070, jitterRate=-0.08104434609413147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:35,736 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:35,737 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for cddeb9d84308524ecd65b2ecb2ba925b: Running coprocessor pre-open hook at 1732329695723Writing region info on filesystem at 1732329695723Initializing all the Stores at 1732329695724 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329695724Cleaning up temporary data from old regions at 1732329695730 (+6 ms)Running coprocessor post-open hooks at 1732329695736 (+6 ms)Region opened successfully at 1732329695737 (+1 ms) 2024-11-23T02:41:35,738 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., pid=27, masterSystemTime=1732329695716 2024-11-23T02:41:35,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,741 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:41:35,741 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=cddeb9d84308524ecd65b2ecb2ba925b, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:35,744 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure cddeb9d84308524ecd65b2ecb2ba925b, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:35,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-23T02:41:35,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure cddeb9d84308524ecd65b2ecb2ba925b, server=72b6db343a5d,43847,1732329616195 in 181 msec 2024-11-23T02:41:35,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-23T02:41:35,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cddeb9d84308524ecd65b2ecb2ba925b, ASSIGN in 340 msec 2024-11-23T02:41:35,751 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:35,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329695751"}]},"ts":"1732329695751"} 2024-11-23T02:41:35,754 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-23T02:41:35,755 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:35,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1830 sec 2024-11-23T02:41:35,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-23T02:41:35,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:35,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-23T02:41:35,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:36,792 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-23T02:41:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T02:41:37,608 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-23T02:41:37,608 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-23T02:41:37,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:37,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-23T02:41:37,613 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:37,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-23T02:41:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:37,626 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:37,626 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-23T02:41:37,629 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T02:41:37,630 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-23T02:41:41,224 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:43,629 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., hostname=72b6db343a5d,43847,1732329616195, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=72b6db343a5d:43847 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-11-23T02:41:43,629 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-11-23T02:41:43,629 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 from cache 2024-11-23T02:41:43,640 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=837 (was 979), ProcessCount=11 (was 11), AvailableMemoryMB=7322 (was 7552) 2024-11-23T02:41:43,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=837, ProcessCount=11, AvailableMemoryMB=7322 2024-11-23T02:41:43,651 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:43,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-23T02:41:43,655 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:43,655 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:43,655 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-23T02:41:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-23T02:41:43,656 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:43,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741851_1027 (size=407) 2024-11-23T02:41:44,067 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1139e58c3a80e0a6d36c61cdac3d2ae4, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:44,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741852_1028 (size=58) 2024-11-23T02:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-23T02:41:44,475 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:44,476 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 1139e58c3a80e0a6d36c61cdac3d2ae4, disabling compactions & flushes 2024-11-23T02:41:44,476 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,476 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,476 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. after waiting 0 ms 2024-11-23T02:41:44,476 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,476 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,476 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1139e58c3a80e0a6d36c61cdac3d2ae4: Waiting for close lock at 1732329704476Disabling compacts and flushes for region at 1732329704476Disabling writes for close at 1732329704476Writing region close event to WAL at 1732329704476Closed at 1732329704476 2024-11-23T02:41:44,478 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:44,478 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732329704478"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329704478"}]},"ts":"1732329704478"} 2024-11-23T02:41:44,481 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:44,482 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:44,482 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329704482"}]},"ts":"1732329704482"} 2024-11-23T02:41:44,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-23T02:41:44,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1139e58c3a80e0a6d36c61cdac3d2ae4, ASSIGN}] 2024-11-23T02:41:44,486 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1139e58c3a80e0a6d36c61cdac3d2ae4, ASSIGN 2024-11-23T02:41:44,487 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1139e58c3a80e0a6d36c61cdac3d2ae4, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:44,556 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T02:41:44,638 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1139e58c3a80e0a6d36c61cdac3d2ae4, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:44,641 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1139e58c3a80e0a6d36c61cdac3d2ae4, ASSIGN because future has completed 2024-11-23T02:41:44,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1139e58c3a80e0a6d36c61cdac3d2ae4, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-23T02:41:44,800 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,800 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1139e58c3a80e0a6d36c61cdac3d2ae4, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:44,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:44,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,803 INFO [StoreOpener-1139e58c3a80e0a6d36c61cdac3d2ae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,805 INFO [StoreOpener-1139e58c3a80e0a6d36c61cdac3d2ae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1139e58c3a80e0a6d36c61cdac3d2ae4 columnFamilyName testFamily 2024-11-23T02:41:44,805 DEBUG [StoreOpener-1139e58c3a80e0a6d36c61cdac3d2ae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:44,805 INFO [StoreOpener-1139e58c3a80e0a6d36c61cdac3d2ae4-1 {}] regionserver.HStore(327): Store=1139e58c3a80e0a6d36c61cdac3d2ae4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:44,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,807 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,807 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,807 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,809 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:44,812 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 1139e58c3a80e0a6d36c61cdac3d2ae4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64253528, jitterRate=-0.04254782199859619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:44,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:44,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1139e58c3a80e0a6d36c61cdac3d2ae4: Running coprocessor pre-open hook at 1732329704801Writing region info on filesystem at 1732329704801Initializing all the Stores at 1732329704802 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329704802Cleaning up temporary data from old regions at 1732329704807 (+5 ms)Running coprocessor post-open hooks at 1732329704813 (+6 ms)Region opened successfully at 1732329704813 2024-11-23T02:41:44,814 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4., pid=30, masterSystemTime=1732329704795 2024-11-23T02:41:44,817 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,817 INFO [RS_OPEN_PRIORITY_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:44,818 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1139e58c3a80e0a6d36c61cdac3d2ae4, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:44,820 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1139e58c3a80e0a6d36c61cdac3d2ae4, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:44,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-23T02:41:44,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1139e58c3a80e0a6d36c61cdac3d2ae4, server=72b6db343a5d,43847,1732329616195 in 180 msec 2024-11-23T02:41:44,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-23T02:41:44,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1139e58c3a80e0a6d36c61cdac3d2ae4, ASSIGN in 339 msec 2024-11-23T02:41:44,828 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:44,828 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329704828"}]},"ts":"1732329704828"} 2024-11-23T02:41:44,831 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-23T02:41:44,832 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:44,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1810 sec 2024-11-23T02:41:45,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-23T02:41:45,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-23T02:41:46,688 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-23T02:41:46,689 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-23T02:41:46,689 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:46,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-23T02:41:46,694 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:46,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-23T02:41:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:46,706 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:46,706 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-23T02:41:46,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-23T02:41:46,711 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-23T02:41:46,712 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-23T02:41:46,712 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-23T02:41:48,794 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-23T02:41:50,302 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:51,526 DEBUG [master/72b6db343a5d:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T02:41:52,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224908' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-23T02:41:52,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60464 deadline: 1732329718723, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224908' does not exist 2024-11-23T02:41:52,724 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 72b6db343a5d,43847,1732329616195 for closing scanner id = -2145723291229224908 for 1139e58c3a80e0a6d36c61cdac3d2ae4 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2145723291229224908' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T02:41:52,730 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=723 (was 837), ProcessCount=11 (was 11), AvailableMemoryMB=7219 (was 7322) 2024-11-23T02:41:52,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=723, ProcessCount=11, AvailableMemoryMB=7218 2024-11-23T02:41:52,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T02:41:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-23T02:41:52,745 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T02:41:52,745 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:52,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-23T02:41:52,746 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T02:41:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T02:41:52,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741853_1029 (size=423) 2024-11-23T02:41:53,157 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0f41bee38fce51ca8708ad12b11fc6f, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e 2024-11-23T02:41:53,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741854_1030 (size=74) 2024-11-23T02:41:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-23T02:41:53,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:53,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing d0f41bee38fce51ca8708ad12b11fc6f, disabling compactions & flushes 2024-11-23T02:41:53,566 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. after waiting 0 ms 2024-11-23T02:41:53,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,566 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0f41bee38fce51ca8708ad12b11fc6f: Waiting for close lock at 1732329713566Disabling compacts and flushes for region at 1732329713566Disabling writes for close at 1732329713566Writing region close event to WAL at 1732329713566Closed at 1732329713566 2024-11-23T02:41:53,568 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T02:41:53,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732329713568"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732329713568"}]},"ts":"1732329713568"} 2024-11-23T02:41:53,571 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T02:41:53,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T02:41:53,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329713572"}]},"ts":"1732329713572"} 2024-11-23T02:41:53,574 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-23T02:41:53,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=d0f41bee38fce51ca8708ad12b11fc6f, ASSIGN}] 2024-11-23T02:41:53,576 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=d0f41bee38fce51ca8708ad12b11fc6f, ASSIGN 2024-11-23T02:41:53,577 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=d0f41bee38fce51ca8708ad12b11fc6f, ASSIGN; state=OFFLINE, location=72b6db343a5d,43847,1732329616195; forceNewPlan=false, retain=false 2024-11-23T02:41:53,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=d0f41bee38fce51ca8708ad12b11fc6f, regionState=OPENING, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:53,731 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=d0f41bee38fce51ca8708ad12b11fc6f, ASSIGN because future has completed 2024-11-23T02:41:53,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0f41bee38fce51ca8708ad12b11fc6f, server=72b6db343a5d,43847,1732329616195}] 2024-11-23T02:41:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T02:41:53,888 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,889 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => d0f41bee38fce51ca8708ad12b11fc6f, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T02:41:53,889 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,889 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T02:41:53,889 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,889 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,891 INFO [StoreOpener-d0f41bee38fce51ca8708ad12b11fc6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,893 INFO [StoreOpener-d0f41bee38fce51ca8708ad12b11fc6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0f41bee38fce51ca8708ad12b11fc6f columnFamilyName testFamily 2024-11-23T02:41:53,893 DEBUG [StoreOpener-d0f41bee38fce51ca8708ad12b11fc6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T02:41:53,893 INFO [StoreOpener-d0f41bee38fce51ca8708ad12b11fc6f-1 {}] regionserver.HStore(327): Store=d0f41bee38fce51ca8708ad12b11fc6f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T02:41:53,893 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,894 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,895 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,895 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,895 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,897 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,900 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T02:41:53,900 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened d0f41bee38fce51ca8708ad12b11fc6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58988681, jitterRate=-0.12100015580654144}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T02:41:53,900 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:53,901 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for d0f41bee38fce51ca8708ad12b11fc6f: Running coprocessor pre-open hook at 1732329713889Writing region info on filesystem at 1732329713889Initializing all the Stores at 1732329713890 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732329713890Cleaning up temporary data from old regions at 1732329713895 (+5 ms)Running coprocessor post-open hooks at 1732329713900 (+5 ms)Region opened successfully at 1732329713901 (+1 ms) 2024-11-23T02:41:53,902 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., pid=33, masterSystemTime=1732329713883 2024-11-23T02:41:53,905 DEBUG [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,905 INFO [RS_OPEN_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:53,905 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=d0f41bee38fce51ca8708ad12b11fc6f, regionState=OPEN, openSeqNum=2, regionLocation=72b6db343a5d,43847,1732329616195 2024-11-23T02:41:53,908 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0f41bee38fce51ca8708ad12b11fc6f, server=72b6db343a5d,43847,1732329616195 because future has completed 2024-11-23T02:41:53,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-23T02:41:53,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure d0f41bee38fce51ca8708ad12b11fc6f, server=72b6db343a5d,43847,1732329616195 in 178 msec 2024-11-23T02:41:53,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-23T02:41:53,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=d0f41bee38fce51ca8708ad12b11fc6f, ASSIGN in 337 msec 2024-11-23T02:41:53,915 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T02:41:53,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732329713915"}]},"ts":"1732329713915"} 2024-11-23T02:41:53,917 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-23T02:41:53,918 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T02:41:53,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1760 sec 2024-11-23T02:41:54,795 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-23T02:41:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T02:41:55,768 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-23T02:41:55,768 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-23T02:41:55,769 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:55,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-23T02:41:55,773 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T02:41:55,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-23T02:41:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:55,784 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-23T02:41:55,785 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-23T02:41:55,787 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-23T02:41:55,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T02:41:55,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-23T02:41:55,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-23T02:41:56,789 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=72b6db343a5d:43847 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-23T02:41:56,790 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-23T02:41:56,790 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 from cache 2024-11-23T02:41:57,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-23T02:41:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2] 2024-11-23T02:41:57,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43847 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-23T02:41:58,789 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=72b6db343a5d:43847 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=978ms, rpcTimeout=976ms 2024-11-23T02:41:58,790 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=978ms, rpcTimeout=976ms 2024-11-23T02:41:58,790 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., hostname=72b6db343a5d,43847,1732329616195, seqNum=2 from cache 2024-11-23T02:41:58,799 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=665 (was 723), ProcessCount=11 (was 11), AvailableMemoryMB=7208 (was 7218) 2024-11-23T02:41:58,799 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T02:41:58,800 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T02:41:58,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:41:58,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:41:58,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T02:41:58,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T02:41:58,805 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T02:41:58,805 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T02:41:58,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:41:58,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:41:58,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T02:41:58,805 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T02:41:58,805 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=688211475, stopped=false 2024-11-23T02:41:58,806 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=72b6db343a5d,43697,1732329615460 2024-11-23T02:41:58,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T02:41:58,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T02:41:58,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:41:58,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:41:58,808 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T02:41:58,808 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T02:41:58,808 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T02:41:58,809 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T02:41:58,809 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T02:41:58,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:41:58,809 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '72b6db343a5d,43847,1732329616195' ***** 2024-11-23T02:41:58,809 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T02:41:58,810 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T02:41:58,810 INFO [RS:0;72b6db343a5d:43847 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T02:41:58,810 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T02:41:58,810 INFO [RS:0;72b6db343a5d:43847 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T02:41:58,810 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d4e3f390252890bbd33ce119c8c6eb3 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 1139e58c3a80e0a6d36c61cdac3d2ae4 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for d0f41bee38fce51ca8708ad12b11fc6f 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 1cf4cc61d248d947709cd85539a97fa8 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 423e69121a389b50524067c0923919d5 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e6f7e4ac90f40f9a72bea324c4ac070 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ddd12496f82516631f13ec470bb219c 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 64f9f4c22f3b6beb764abd436719011a 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(959): stopping server 72b6db343a5d,43847,1732329616195 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T02:41:58,811 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d4e3f390252890bbd33ce119c8c6eb3, disabling compactions & flushes 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;72b6db343a5d:43847. 2024-11-23T02:41:58,811 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:41:58,811 DEBUG [RS:0;72b6db343a5d:43847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T02:41:58,811 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:41:58,811 DEBUG [RS:0;72b6db343a5d:43847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:41:58,811 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. after waiting 0 ms 2024-11-23T02:41:58,811 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T02:41:58,811 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T02:41:58,812 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T02:41:58,812 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-23T02:41:58,812 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T02:41:58,812 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T02:41:58,812 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1139e58c3a80e0a6d36c61cdac3d2ae4, 1588230740, 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4d4e3f390252890bbd33ce119c8c6eb3, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:58,812 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T02:41:58,812 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T02:41:58,812 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T02:41:58,815 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4d4e3f390252890bbd33ce119c8c6eb3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:41:58,816 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-23T02:41:58,873 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/.tmp/testFamily/6d8a370111d146cc906f22ea6304e610 is 61, key is row-0/testFamily:testQualifier/1732329660042/Put/seqid=0 2024-11-23T02:41:58,879 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/info/9d22ae7967664a6c9ba3cd66c932486a is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070./info:regioninfo/1732329654230/Put/seqid=0 2024-11-23T02:41:58,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741855_1031 (size=5049) 2024-11-23T02:41:58,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741856_1032 (size=20564) 2024-11-23T02:41:58,886 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/info/9d22ae7967664a6c9ba3cd66c932486a 2024-11-23T02:41:58,955 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/ns/04cc3013e8cb44ac9db32b8dcb4c6565 is 43, key is default/ns:d/1732329651160/Put/seqid=0 2024-11-23T02:41:58,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741857_1033 (size=5153) 2024-11-23T02:41:59,012 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1139e58c3a80e0a6d36c61cdac3d2ae4, 1588230740, 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4d4e3f390252890bbd33ce119c8c6eb3, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:59,213 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1139e58c3a80e0a6d36c61cdac3d2ae4, 1588230740, 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4d4e3f390252890bbd33ce119c8c6eb3, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:59,284 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/.tmp/testFamily/6d8a370111d146cc906f22ea6304e610 2024-11-23T02:41:59,294 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/.tmp/testFamily/6d8a370111d146cc906f22ea6304e610 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/testFamily/6d8a370111d146cc906f22ea6304e610 2024-11-23T02:41:59,301 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/testFamily/6d8a370111d146cc906f22ea6304e610, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:41:59,311 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4d4e3f390252890bbd33ce119c8c6eb3 in 494ms, sequenceid=8, compaction requested=false 2024-11-23T02:41:59,312 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-23T02:41:59,317 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScanner/4d4e3f390252890bbd33ce119c8c6eb3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:41:59,319 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:41:59,319 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d4e3f390252890bbd33ce119c8c6eb3: Waiting for close lock at 1732329718811Running coprocessor pre-close hooks at 1732329718811Disabling compacts and flushes for region at 1732329718811Disabling writes for close at 1732329718811Obtaining lock to block concurrent updates at 1732329718815 (+4 ms)Preparing flush snapshotting stores in 4d4e3f390252890bbd33ce119c8c6eb3 at 1732329718816 (+1 ms)Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329718823 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. at 1732329718825 (+2 ms)Flushing 4d4e3f390252890bbd33ce119c8c6eb3/testFamily: creating writer at 1732329718826 (+1 ms)Flushing 4d4e3f390252890bbd33ce119c8c6eb3/testFamily: appending metadata at 1732329718867 (+41 ms)Flushing 4d4e3f390252890bbd33ce119c8c6eb3/testFamily: closing flushed file at 1732329718870 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad2967b: reopening flushed file at 1732329719293 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4d4e3f390252890bbd33ce119c8c6eb3 in 494ms, sequenceid=8, compaction requested=false at 1732329719311 (+18 ms)Writing region close event to WAL at 1732329719312 (+1 ms)Running coprocessor post-close hooks at 1732329719317 (+5 ms)Closed at 1732329719319 (+2 ms) 2024-11-23T02:41:59,319 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732329656923.4d4e3f390252890bbd33ce119c8c6eb3. 2024-11-23T02:41:59,320 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1139e58c3a80e0a6d36c61cdac3d2ae4, disabling compactions & flushes 2024-11-23T02:41:59,320 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:59,320 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:59,320 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. after waiting 0 ms 2024-11-23T02:41:59,320 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:59,320 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1139e58c3a80e0a6d36c61cdac3d2ae4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:41:59,337 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/.tmp/testFamily/4aac6cbbc8194208ac4c3038ae8e1ec8 is 61, key is row-0/testFamily:testQualifier/1732329706697/Put/seqid=0 2024-11-23T02:41:59,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741858_1034 (size=5049) 2024-11-23T02:41:59,361 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/ns/04cc3013e8cb44ac9db32b8dcb4c6565 2024-11-23T02:41:59,385 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/table/3aaf4f1f91514b0599072facb7c35b44 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732329654376/Put/seqid=0 2024-11-23T02:41:59,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741859_1035 (size=6699) 2024-11-23T02:41:59,390 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:41:59,413 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1139e58c3a80e0a6d36c61cdac3d2ae4, 1588230740, 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:59,613 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1139e58c3a80e0a6d36c61cdac3d2ae4, 1588230740, 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:59,650 INFO [regionserver/72b6db343a5d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T02:41:59,650 INFO [regionserver/72b6db343a5d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T02:41:59,742 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/.tmp/testFamily/4aac6cbbc8194208ac4c3038ae8e1ec8 2024-11-23T02:41:59,752 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/.tmp/testFamily/4aac6cbbc8194208ac4c3038ae8e1ec8 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/testFamily/4aac6cbbc8194208ac4c3038ae8e1ec8 2024-11-23T02:41:59,759 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/testFamily/4aac6cbbc8194208ac4c3038ae8e1ec8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:41:59,761 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1139e58c3a80e0a6d36c61cdac3d2ae4 in 440ms, sequenceid=8, compaction requested=false 2024-11-23T02:41:59,765 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNext/1139e58c3a80e0a6d36c61cdac3d2ae4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:41:59,766 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:59,766 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1139e58c3a80e0a6d36c61cdac3d2ae4: Waiting for close lock at 1732329719320Running coprocessor pre-close hooks at 1732329719320Disabling compacts and flushes for region at 1732329719320Disabling writes for close at 1732329719320Obtaining lock to block concurrent updates at 1732329719320Preparing flush snapshotting stores in 1139e58c3a80e0a6d36c61cdac3d2ae4 at 1732329719320Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329719320Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. at 1732329719321 (+1 ms)Flushing 1139e58c3a80e0a6d36c61cdac3d2ae4/testFamily: creating writer at 1732329719321Flushing 1139e58c3a80e0a6d36c61cdac3d2ae4/testFamily: appending metadata at 1732329719336 (+15 ms)Flushing 1139e58c3a80e0a6d36c61cdac3d2ae4/testFamily: closing flushed file at 1732329719337 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@264220ad: reopening flushed file at 1732329719750 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1139e58c3a80e0a6d36c61cdac3d2ae4 in 440ms, sequenceid=8, compaction requested=false at 1732329719761 (+11 ms)Writing region close event to WAL at 1732329719761Running coprocessor post-close hooks at 1732329719766 (+5 ms)Closed at 1732329719766 2024-11-23T02:41:59,766 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732329703651.1139e58c3a80e0a6d36c61cdac3d2ae4. 2024-11-23T02:41:59,767 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d0f41bee38fce51ca8708ad12b11fc6f, disabling compactions & flushes 2024-11-23T02:41:59,767 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:59,767 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:59,767 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. after waiting 0 ms 2024-11-23T02:41:59,767 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:41:59,767 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d0f41bee38fce51ca8708ad12b11fc6f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:41:59,783 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/.tmp/testFamily/02de3e5d017349a7af74654c1f963a96 is 61, key is row-0/testFamily:testQualifier/1732329715777/Put/seqid=0 2024-11-23T02:41:59,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741860_1036 (size=5049) 2024-11-23T02:41:59,791 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/table/3aaf4f1f91514b0599072facb7c35b44 2024-11-23T02:41:59,798 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/info/9d22ae7967664a6c9ba3cd66c932486a as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/info/9d22ae7967664a6c9ba3cd66c932486a 2024-11-23T02:41:59,806 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/info/9d22ae7967664a6c9ba3cd66c932486a, entries=100, sequenceid=56, filesize=20.1 K 2024-11-23T02:41:59,807 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/ns/04cc3013e8cb44ac9db32b8dcb4c6565 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/ns/04cc3013e8cb44ac9db32b8dcb4c6565 2024-11-23T02:41:59,813 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-23T02:41:59,814 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1325): Online Regions={d0f41bee38fce51ca8708ad12b11fc6f=testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., 1cf4cc61d248d947709cd85539a97fa8=testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8., 423e69121a389b50524067c0923919d5=testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5., 1588230740=hbase:meta,,1.1588230740, 4e6f7e4ac90f40f9a72bea324c4ac070=testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070., e6cf9b9df16853f5abc52ca3755323dd=testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd., 2ddd12496f82516631f13ec470bb219c=hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c., 64f9f4c22f3b6beb764abd436719011a=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., cddeb9d84308524ecd65b2ecb2ba925b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.} 2024-11-23T02:41:59,814 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:41:59,814 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/ns/04cc3013e8cb44ac9db32b8dcb4c6565, entries=2, sequenceid=56, filesize=5.0 K 2024-11-23T02:41:59,815 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/.tmp/table/3aaf4f1f91514b0599072facb7c35b44 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/table/3aaf4f1f91514b0599072facb7c35b44 2024-11-23T02:41:59,822 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/table/3aaf4f1f91514b0599072facb7c35b44, entries=20, sequenceid=56, filesize=6.5 K 2024-11-23T02:41:59,823 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1011ms, sequenceid=56, compaction requested=false 2024-11-23T02:41:59,828 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-23T02:41:59,829 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T02:41:59,829 INFO [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T02:41:59,829 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732329718812Running coprocessor pre-close hooks at 1732329718812Disabling compacts and flushes for region at 1732329718812Disabling writes for close at 1732329718812Obtaining lock to block concurrent updates at 1732329718816 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1732329718816Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732329718824 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732329718825 (+1 ms)Flushing 1588230740/info: creating writer at 1732329718826 (+1 ms)Flushing 1588230740/info: appending metadata at 1732329718875 (+49 ms)Flushing 1588230740/info: closing flushed file at 1732329718875Flushing 1588230740/ns: creating writer at 1732329718937 (+62 ms)Flushing 1588230740/ns: appending metadata at 1732329718954 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732329718954Flushing 1588230740/table: creating writer at 1732329719369 (+415 ms)Flushing 1588230740/table: appending metadata at 1732329719384 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732329719384Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe525a6: reopening flushed file at 1732329719797 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6c7a57: reopening flushed file at 1732329719806 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a92c8d: reopening flushed file at 1732329719814 (+8 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1011ms, sequenceid=56, compaction requested=false at 1732329719823 (+9 ms)Writing region close event to WAL at 1732329719824 (+1 ms)Running coprocessor post-close hooks at 1732329719828 (+4 ms)Closed at 1732329719829 (+1 ms) 2024-11-23T02:41:59,829 DEBUG [RS_CLOSE_META-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T02:42:00,014 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, d0f41bee38fce51ca8708ad12b11fc6f, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:00,189 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/.tmp/testFamily/02de3e5d017349a7af74654c1f963a96 2024-11-23T02:42:00,197 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/.tmp/testFamily/02de3e5d017349a7af74654c1f963a96 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/testFamily/02de3e5d017349a7af74654c1f963a96 2024-11-23T02:42:00,204 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/testFamily/02de3e5d017349a7af74654c1f963a96, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:00,205 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d0f41bee38fce51ca8708ad12b11fc6f in 438ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:00,210 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScannerAsync/d0f41bee38fce51ca8708ad12b11fc6f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:00,211 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:42:00,211 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d0f41bee38fce51ca8708ad12b11fc6f: Waiting for close lock at 1732329719766Running coprocessor pre-close hooks at 1732329719766Disabling compacts and flushes for region at 1732329719766Disabling writes for close at 1732329719767 (+1 ms)Obtaining lock to block concurrent updates at 1732329719767Preparing flush snapshotting stores in d0f41bee38fce51ca8708ad12b11fc6f at 1732329719767Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329719767Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. at 1732329719768 (+1 ms)Flushing d0f41bee38fce51ca8708ad12b11fc6f/testFamily: creating writer at 1732329719768Flushing d0f41bee38fce51ca8708ad12b11fc6f/testFamily: appending metadata at 1732329719783 (+15 ms)Flushing d0f41bee38fce51ca8708ad12b11fc6f/testFamily: closing flushed file at 1732329719783Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49fbbcaa: reopening flushed file at 1732329720196 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d0f41bee38fce51ca8708ad12b11fc6f in 438ms, sequenceid=8, compaction requested=false at 1732329720205 (+9 ms)Writing region close event to WAL at 1732329720206 (+1 ms)Running coprocessor post-close hooks at 1732329720211 (+5 ms)Closed at 1732329720211 2024-11-23T02:42:00,211 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732329712742.d0f41bee38fce51ca8708ad12b11fc6f. 2024-11-23T02:42:00,211 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1cf4cc61d248d947709cd85539a97fa8, disabling compactions & flushes 2024-11-23T02:42:00,211 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:42:00,211 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:42:00,211 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. after waiting 0 ms 2024-11-23T02:42:00,211 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:42:00,211 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1cf4cc61d248d947709cd85539a97fa8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:00,214 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:00,228 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/.tmp/testFamily/3c7092b448374c9a8d53abb30bf64824 is 61, key is row-0/testFamily:testQualifier/1732329676238/Put/seqid=0 2024-11-23T02:42:00,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741861_1037 (size=5049) 2024-11-23T02:42:00,414 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:00,420 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=72b6db343a5d:43697 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 10 more 2024-11-23T02:42:00,615 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 1cf4cc61d248d947709cd85539a97fa8, 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:00,634 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/.tmp/testFamily/3c7092b448374c9a8d53abb30bf64824 2024-11-23T02:42:00,642 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/.tmp/testFamily/3c7092b448374c9a8d53abb30bf64824 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/testFamily/3c7092b448374c9a8d53abb30bf64824 2024-11-23T02:42:00,650 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/testFamily/3c7092b448374c9a8d53abb30bf64824, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:00,651 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1cf4cc61d248d947709cd85539a97fa8 in 440ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:00,656 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextException/1cf4cc61d248d947709cd85539a97fa8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:00,657 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:42:00,657 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1cf4cc61d248d947709cd85539a97fa8: Waiting for close lock at 1732329720211Running coprocessor pre-close hooks at 1732329720211Disabling compacts and flushes for region at 1732329720211Disabling writes for close at 1732329720211Obtaining lock to block concurrent updates at 1732329720211Preparing flush snapshotting stores in 1cf4cc61d248d947709cd85539a97fa8 at 1732329720211Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329720212 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. at 1732329720212Flushing 1cf4cc61d248d947709cd85539a97fa8/testFamily: creating writer at 1732329720212Flushing 1cf4cc61d248d947709cd85539a97fa8/testFamily: appending metadata at 1732329720228 (+16 ms)Flushing 1cf4cc61d248d947709cd85539a97fa8/testFamily: closing flushed file at 1732329720228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e4f4f5b: reopening flushed file at 1732329720641 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1cf4cc61d248d947709cd85539a97fa8 in 440ms, sequenceid=8, compaction requested=false at 1732329720651 (+10 ms)Writing region close event to WAL at 1732329720652 (+1 ms)Running coprocessor post-close hooks at 1732329720657 (+5 ms)Closed at 1732329720657 2024-11-23T02:42:00,657 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732329675214.1cf4cc61d248d947709cd85539a97fa8. 2024-11-23T02:42:00,657 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 423e69121a389b50524067c0923919d5, disabling compactions & flushes 2024-11-23T02:42:00,657 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:42:00,658 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:42:00,658 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. after waiting 0 ms 2024-11-23T02:42:00,658 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:42:00,658 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 423e69121a389b50524067c0923919d5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:00,681 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/.tmp/testFamily/eb2b15eb04bf4f6c929594d3c4e03584 is 61, key is row-0/testFamily:testQualifier/1732329691528/Put/seqid=0 2024-11-23T02:42:00,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741862_1038 (size=5049) 2024-11-23T02:42:00,815 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-23T02:42:00,815 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1325): Online Regions={423e69121a389b50524067c0923919d5=testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5., 4e6f7e4ac90f40f9a72bea324c4ac070=testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070., e6cf9b9df16853f5abc52ca3755323dd=testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd., 2ddd12496f82516631f13ec470bb219c=hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c., 64f9f4c22f3b6beb764abd436719011a=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., cddeb9d84308524ecd65b2ecb2ba925b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.} 2024-11-23T02:42:00,815 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:01,015 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 423e69121a389b50524067c0923919d5, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:01,087 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/.tmp/testFamily/eb2b15eb04bf4f6c929594d3c4e03584 2024-11-23T02:42:01,097 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/.tmp/testFamily/eb2b15eb04bf4f6c929594d3c4e03584 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/testFamily/eb2b15eb04bf4f6c929594d3c4e03584 2024-11-23T02:42:01,104 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/testFamily/eb2b15eb04bf4f6c929594d3c4e03584, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:01,106 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 423e69121a389b50524067c0923919d5 in 448ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:01,111 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNext/423e69121a389b50524067c0923919d5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:01,111 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:42:01,112 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 423e69121a389b50524067c0923919d5: Waiting for close lock at 1732329720657Running coprocessor pre-close hooks at 1732329720657Disabling compacts and flushes for region at 1732329720657Disabling writes for close at 1732329720658 (+1 ms)Obtaining lock to block concurrent updates at 1732329720658Preparing flush snapshotting stores in 423e69121a389b50524067c0923919d5 at 1732329720658Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329720658Flushing stores of testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. at 1732329720659 (+1 ms)Flushing 423e69121a389b50524067c0923919d5/testFamily: creating writer at 1732329720659Flushing 423e69121a389b50524067c0923919d5/testFamily: appending metadata at 1732329720681 (+22 ms)Flushing 423e69121a389b50524067c0923919d5/testFamily: closing flushed file at 1732329720681Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@665dd426: reopening flushed file at 1732329721095 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 423e69121a389b50524067c0923919d5 in 448ms, sequenceid=8, compaction requested=false at 1732329721106 (+11 ms)Writing region close event to WAL at 1732329721107 (+1 ms)Running coprocessor post-close hooks at 1732329721111 (+4 ms)Closed at 1732329721111 2024-11-23T02:42:01,112 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732329688482.423e69121a389b50524067c0923919d5. 2024-11-23T02:42:01,112 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e6f7e4ac90f40f9a72bea324c4ac070, disabling compactions & flushes 2024-11-23T02:42:01,112 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:42:01,112 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:42:01,112 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. after waiting 0 ms 2024-11-23T02:42:01,112 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:42:01,112 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4e6f7e4ac90f40f9a72bea324c4ac070 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:01,129 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/.tmp/testFamily/f3bb79589f34428c90229b536a018dc2 is 61, key is row-0/testFamily:testQualifier/1732329656141/Put/seqid=0 2024-11-23T02:42:01,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741863_1039 (size=5049) 2024-11-23T02:42:01,216 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:01,416 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 4e6f7e4ac90f40f9a72bea324c4ac070, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:01,534 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/.tmp/testFamily/f3bb79589f34428c90229b536a018dc2 2024-11-23T02:42:01,542 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/.tmp/testFamily/f3bb79589f34428c90229b536a018dc2 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/testFamily/f3bb79589f34428c90229b536a018dc2 2024-11-23T02:42:01,551 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/testFamily/f3bb79589f34428c90229b536a018dc2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:01,552 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4e6f7e4ac90f40f9a72bea324c4ac070 in 440ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:01,557 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e6f7e4ac90f40f9a72bea324c4ac070/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:01,558 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:42:01,558 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e6f7e4ac90f40f9a72bea324c4ac070: Waiting for close lock at 1732329721112Running coprocessor pre-close hooks at 1732329721112Disabling compacts and flushes for region at 1732329721112Disabling writes for close at 1732329721112Obtaining lock to block concurrent updates at 1732329721112Preparing flush snapshotting stores in 4e6f7e4ac90f40f9a72bea324c4ac070 at 1732329721112Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329721113 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. at 1732329721113Flushing 4e6f7e4ac90f40f9a72bea324c4ac070/testFamily: creating writer at 1732329721113Flushing 4e6f7e4ac90f40f9a72bea324c4ac070/testFamily: appending metadata at 1732329721128 (+15 ms)Flushing 4e6f7e4ac90f40f9a72bea324c4ac070/testFamily: closing flushed file at 1732329721128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44b6f226: reopening flushed file at 1732329721541 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4e6f7e4ac90f40f9a72bea324c4ac070 in 440ms, sequenceid=8, compaction requested=false at 1732329721552 (+11 ms)Writing region close event to WAL at 1732329721553 (+1 ms)Running coprocessor post-close hooks at 1732329721558 (+5 ms)Closed at 1732329721558 2024-11-23T02:42:01,558 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732329652692.4e6f7e4ac90f40f9a72bea324c4ac070. 2024-11-23T02:42:01,559 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e6cf9b9df16853f5abc52ca3755323dd, disabling compactions & flushes 2024-11-23T02:42:01,559 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:42:01,559 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:42:01,559 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. after waiting 0 ms 2024-11-23T02:42:01,559 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:42:01,559 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e6cf9b9df16853f5abc52ca3755323dd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:01,576 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/.tmp/testFamily/e5aa52d3194441b88f69df8df43b2c01 is 61, key is row-0/testFamily:testQualifier/1732329685437/Put/seqid=0 2024-11-23T02:42:01,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741864_1040 (size=5049) 2024-11-23T02:42:01,616 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:01,816 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-23T02:42:01,817 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1325): Online Regions={e6cf9b9df16853f5abc52ca3755323dd=testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd., 2ddd12496f82516631f13ec470bb219c=hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c., 64f9f4c22f3b6beb764abd436719011a=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., cddeb9d84308524ecd65b2ecb2ba925b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.} 2024-11-23T02:42:01,817 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b, e6cf9b9df16853f5abc52ca3755323dd 2024-11-23T02:42:01,982 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/.tmp/testFamily/e5aa52d3194441b88f69df8df43b2c01 2024-11-23T02:42:01,991 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/.tmp/testFamily/e5aa52d3194441b88f69df8df43b2c01 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/testFamily/e5aa52d3194441b88f69df8df43b2c01 2024-11-23T02:42:01,998 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/testFamily/e5aa52d3194441b88f69df8df43b2c01, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:01,999 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e6cf9b9df16853f5abc52ca3755323dd in 440ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:02,003 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnNextAsync/e6cf9b9df16853f5abc52ca3755323dd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:02,004 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:42:02,004 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e6cf9b9df16853f5abc52ca3755323dd: Waiting for close lock at 1732329721559Running coprocessor pre-close hooks at 1732329721559Disabling compacts and flushes for region at 1732329721559Disabling writes for close at 1732329721559Obtaining lock to block concurrent updates at 1732329721559Preparing flush snapshotting stores in e6cf9b9df16853f5abc52ca3755323dd at 1732329721559Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329721559Flushing stores of testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. at 1732329721560 (+1 ms)Flushing e6cf9b9df16853f5abc52ca3755323dd/testFamily: creating writer at 1732329721560Flushing e6cf9b9df16853f5abc52ca3755323dd/testFamily: appending metadata at 1732329721576 (+16 ms)Flushing e6cf9b9df16853f5abc52ca3755323dd/testFamily: closing flushed file at 1732329721576Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67fee542: reopening flushed file at 1732329721990 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e6cf9b9df16853f5abc52ca3755323dd in 440ms, sequenceid=8, compaction requested=false at 1732329721999 (+9 ms)Writing region close event to WAL at 1732329722000 (+1 ms)Running coprocessor post-close hooks at 1732329722004 (+4 ms)Closed at 1732329722004 2024-11-23T02:42:02,004 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732329682379.e6cf9b9df16853f5abc52ca3755323dd. 2024-11-23T02:42:02,004 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ddd12496f82516631f13ec470bb219c, disabling compactions & flushes 2024-11-23T02:42:02,004 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:42:02,005 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:42:02,005 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. after waiting 0 ms 2024-11-23T02:42:02,005 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:42:02,005 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2ddd12496f82516631f13ec470bb219c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:02,017 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:02,020 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/.tmp/testFamily/a75c6862c6244bb8bd5d8918788eecc6 is 61, key is row-0/testFamily:testQualifier/1732329669170/Put/seqid=0 2024-11-23T02:42:02,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741865_1041 (size=5049) 2024-11-23T02:42:02,217 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:02,417 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 2ddd12496f82516631f13ec470bb219c, 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:02,426 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/.tmp/testFamily/a75c6862c6244bb8bd5d8918788eecc6 2024-11-23T02:42:02,434 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/.tmp/testFamily/a75c6862c6244bb8bd5d8918788eecc6 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/testFamily/a75c6862c6244bb8bd5d8918788eecc6 2024-11-23T02:42:02,441 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/testFamily/a75c6862c6244bb8bd5d8918788eecc6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:02,442 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ddd12496f82516631f13ec470bb219c in 437ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:02,447 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnNextAsync/2ddd12496f82516631f13ec470bb219c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:02,448 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:42:02,448 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ddd12496f82516631f13ec470bb219c: Waiting for close lock at 1732329722004Running coprocessor pre-close hooks at 1732329722004Disabling compacts and flushes for region at 1732329722004Disabling writes for close at 1732329722005 (+1 ms)Obtaining lock to block concurrent updates at 1732329722005Preparing flush snapshotting stores in 2ddd12496f82516631f13ec470bb219c at 1732329722005Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329722005Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. at 1732329722005Flushing 2ddd12496f82516631f13ec470bb219c/testFamily: creating writer at 1732329722005Flushing 2ddd12496f82516631f13ec470bb219c/testFamily: appending metadata at 1732329722020 (+15 ms)Flushing 2ddd12496f82516631f13ec470bb219c/testFamily: closing flushed file at 1732329722020Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c44468: reopening flushed file at 1732329722433 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ddd12496f82516631f13ec470bb219c in 437ms, sequenceid=8, compaction requested=false at 1732329722442 (+9 ms)Writing region close event to WAL at 1732329722443 (+1 ms)Running coprocessor post-close hooks at 1732329722448 (+5 ms)Closed at 1732329722448 2024-11-23T02:42:02,448 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732329666107.2ddd12496f82516631f13ec470bb219c. 2024-11-23T02:42:02,448 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64f9f4c22f3b6beb764abd436719011a, disabling compactions & flushes 2024-11-23T02:42:02,448 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:42:02,448 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:42:02,448 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. after waiting 0 ms 2024-11-23T02:42:02,448 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:42:02,449 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 64f9f4c22f3b6beb764abd436719011a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:02,465 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/.tmp/testFamily/87ecae1170aa4f089efc6a513c2c648c is 61, key is row-0/testFamily:testQualifier/1732329679339/Put/seqid=0 2024-11-23T02:42:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741866_1042 (size=5049) 2024-11-23T02:42:02,618 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:02,818 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T02:42:02,818 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1325): Online Regions={64f9f4c22f3b6beb764abd436719011a=testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., cddeb9d84308524ecd65b2ecb2ba925b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b.} 2024-11-23T02:42:02,818 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on 64f9f4c22f3b6beb764abd436719011a, cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:02,871 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/.tmp/testFamily/87ecae1170aa4f089efc6a513c2c648c 2024-11-23T02:42:02,879 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/.tmp/testFamily/87ecae1170aa4f089efc6a513c2c648c as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/testFamily/87ecae1170aa4f089efc6a513c2c648c 2024-11-23T02:42:02,886 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/testFamily/87ecae1170aa4f089efc6a513c2c648c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:02,888 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 64f9f4c22f3b6beb764abd436719011a in 438ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:02,892 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/default/testNormalScanTimeoutOnOpenScanner/64f9f4c22f3b6beb764abd436719011a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:02,893 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:42:02,893 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64f9f4c22f3b6beb764abd436719011a: Waiting for close lock at 1732329722448Running coprocessor pre-close hooks at 1732329722448Disabling compacts and flushes for region at 1732329722448Disabling writes for close at 1732329722448Obtaining lock to block concurrent updates at 1732329722449 (+1 ms)Preparing flush snapshotting stores in 64f9f4c22f3b6beb764abd436719011a at 1732329722449Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329722449Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. at 1732329722449Flushing 64f9f4c22f3b6beb764abd436719011a/testFamily: creating writer at 1732329722450 (+1 ms)Flushing 64f9f4c22f3b6beb764abd436719011a/testFamily: appending metadata at 1732329722465 (+15 ms)Flushing 64f9f4c22f3b6beb764abd436719011a/testFamily: closing flushed file at 1732329722465Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ba3b6f9: reopening flushed file at 1732329722878 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 64f9f4c22f3b6beb764abd436719011a in 438ms, sequenceid=8, compaction requested=false at 1732329722888 (+10 ms)Writing region close event to WAL at 1732329722888Running coprocessor post-close hooks at 1732329722893 (+5 ms)Closed at 1732329722893 2024-11-23T02:42:02,893 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732329676301.64f9f4c22f3b6beb764abd436719011a. 2024-11-23T02:42:02,893 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cddeb9d84308524ecd65b2ecb2ba925b, disabling compactions & flushes 2024-11-23T02:42:02,893 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:42:02,893 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:42:02,893 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. after waiting 0 ms 2024-11-23T02:42:02,894 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:42:02,894 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cddeb9d84308524ecd65b2ecb2ba925b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-23T02:42:02,910 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/.tmp/testFamily/72320558f502471a8aec884906130583 is 61, key is row-0/testFamily:testQualifier/1732329697617/Put/seqid=0 2024-11-23T02:42:02,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741867_1043 (size=5049) 2024-11-23T02:42:03,018 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:03,219 DEBUG [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1351): Waiting on cddeb9d84308524ecd65b2ecb2ba925b 2024-11-23T02:42:03,316 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/.tmp/testFamily/72320558f502471a8aec884906130583 2024-11-23T02:42:03,324 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/.tmp/testFamily/72320558f502471a8aec884906130583 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/testFamily/72320558f502471a8aec884906130583 2024-11-23T02:42:03,331 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/testFamily/72320558f502471a8aec884906130583, entries=4, sequenceid=8, filesize=4.9 K 2024-11-23T02:42:03,332 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cddeb9d84308524ecd65b2ecb2ba925b in 438ms, sequenceid=8, compaction requested=false 2024-11-23T02:42:03,337 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cddeb9d84308524ecd65b2ecb2ba925b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-23T02:42:03,337 INFO [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:42:03,338 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cddeb9d84308524ecd65b2ecb2ba925b: Waiting for close lock at 1732329722893Running coprocessor pre-close hooks at 1732329722893Disabling compacts and flushes for region at 1732329722893Disabling writes for close at 1732329722893Obtaining lock to block concurrent updates at 1732329722894 (+1 ms)Preparing flush snapshotting stores in cddeb9d84308524ecd65b2ecb2ba925b at 1732329722894Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732329722894Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. at 1732329722894Flushing cddeb9d84308524ecd65b2ecb2ba925b/testFamily: creating writer at 1732329722895 (+1 ms)Flushing cddeb9d84308524ecd65b2ecb2ba925b/testFamily: appending metadata at 1732329722910 (+15 ms)Flushing cddeb9d84308524ecd65b2ecb2ba925b/testFamily: closing flushed file at 1732329722910Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a8b6b70: reopening flushed file at 1732329723323 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cddeb9d84308524ecd65b2ecb2ba925b in 438ms, sequenceid=8, compaction requested=false at 1732329723332 (+9 ms)Writing region close event to WAL at 1732329723333 (+1 ms)Running coprocessor post-close hooks at 1732329723337 (+4 ms)Closed at 1732329723337 2024-11-23T02:42:03,338 DEBUG [RS_CLOSE_REGION-regionserver/72b6db343a5d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732329694571.cddeb9d84308524ecd65b2ecb2ba925b. 2024-11-23T02:42:03,419 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(976): stopping server 72b6db343a5d,43847,1732329616195; all regions closed. 2024-11-23T02:42:03,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741834_1010 (size=19479) 2024-11-23T02:42:03,430 DEBUG [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/oldWALs 2024-11-23T02:42:03,430 INFO [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 72b6db343a5d%2C43847%2C1732329616195.meta:.meta(num 1732329650084) 2024-11-23T02:42:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741833_1009 (size=20031) 2024-11-23T02:42:03,433 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/WALs/72b6db343a5d,43847,1732329616195/72b6db343a5d%2C43847%2C1732329616195.1732329648682 not finished, retry = 0 2024-11-23T02:42:03,536 DEBUG [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/oldWALs 2024-11-23T02:42:03,536 INFO [RS:0;72b6db343a5d:43847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 72b6db343a5d%2C43847%2C1732329616195:(num 1732329648682) 2024-11-23T02:42:03,536 DEBUG [RS:0;72b6db343a5d:43847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T02:42:03,536 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T02:42:03,536 INFO [RS:0;72b6db343a5d:43847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T02:42:03,536 INFO [RS:0;72b6db343a5d:43847 {}] hbase.ChoreService(370): Chore service for: regionserver/72b6db343a5d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T02:42:03,537 INFO [RS:0;72b6db343a5d:43847 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T02:42:03,537 INFO [regionserver/72b6db343a5d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T02:42:03,538 INFO [RS:0;72b6db343a5d:43847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43847 2024-11-23T02:42:03,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/72b6db343a5d,43847,1732329616195 2024-11-23T02:42:03,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T02:42:03,542 INFO [RS:0;72b6db343a5d:43847 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T02:42:03,544 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [72b6db343a5d,43847,1732329616195] 2024-11-23T02:42:03,545 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/72b6db343a5d,43847,1732329616195 already deleted, retry=false 2024-11-23T02:42:03,546 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 72b6db343a5d,43847,1732329616195 expired; onlineServers=0 2024-11-23T02:42:03,546 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '72b6db343a5d,43697,1732329615460' ***** 2024-11-23T02:42:03,546 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T02:42:03,546 INFO [M:0;72b6db343a5d:43697 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T02:42:03,546 INFO [M:0;72b6db343a5d:43697 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T02:42:03,546 DEBUG [M:0;72b6db343a5d:43697 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T02:42:03,546 DEBUG [M:0;72b6db343a5d:43697 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T02:42:03,546 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T02:42:03,546 DEBUG [master/72b6db343a5d:0:becomeActiveMaster-HFileCleaner.small.0-1732329618438 {}] cleaner.HFileCleaner(306): Exit Thread[master/72b6db343a5d:0:becomeActiveMaster-HFileCleaner.small.0-1732329618438,5,FailOnTimeoutGroup] 2024-11-23T02:42:03,546 DEBUG [master/72b6db343a5d:0:becomeActiveMaster-HFileCleaner.large.0-1732329618437 {}] cleaner.HFileCleaner(306): Exit Thread[master/72b6db343a5d:0:becomeActiveMaster-HFileCleaner.large.0-1732329618437,5,FailOnTimeoutGroup] 2024-11-23T02:42:03,547 INFO [M:0;72b6db343a5d:43697 {}] hbase.ChoreService(370): Chore service for: master/72b6db343a5d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T02:42:03,547 INFO [M:0;72b6db343a5d:43697 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T02:42:03,547 DEBUG [M:0;72b6db343a5d:43697 {}] master.HMaster(1795): Stopping service threads 2024-11-23T02:42:03,547 INFO [M:0;72b6db343a5d:43697 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T02:42:03,547 INFO [M:0;72b6db343a5d:43697 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T02:42:03,547 INFO [M:0;72b6db343a5d:43697 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T02:42:03,548 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T02:42:03,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T02:42:03,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T02:42:03,549 DEBUG [M:0;72b6db343a5d:43697 {}] zookeeper.ZKUtil(347): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T02:42:03,549 WARN [M:0;72b6db343a5d:43697 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T02:42:03,550 INFO [M:0;72b6db343a5d:43697 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/.lastflushedseqids 2024-11-23T02:42:03,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741868_1044 (size=649) 2024-11-23T02:42:03,621 INFO [regionserver/72b6db343a5d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T02:42:03,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T02:42:03,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43847-0x10142b217ae0001, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T02:42:03,645 INFO [RS:0;72b6db343a5d:43847 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T02:42:03,645 INFO [RS:0;72b6db343a5d:43847 {}] regionserver.HRegionServer(1031): Exiting; stopping=72b6db343a5d,43847,1732329616195; zookeeper connection closed. 2024-11-23T02:42:03,645 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55accc1e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55accc1e 2024-11-23T02:42:03,646 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T02:42:03,962 INFO [M:0;72b6db343a5d:43697 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T02:42:03,962 INFO [M:0;72b6db343a5d:43697 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T02:42:03,962 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T02:42:03,963 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:42:03,963 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:42:03,963 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T02:42:03,963 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:42:03,963 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.96 KB heapSize=190.83 KB 2024-11-23T02:42:03,980 DEBUG [M:0;72b6db343a5d:43697 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/df634f57c20148ea9dd51a217e2d0b5e is 82, key is hbase:meta,,1/info:regioninfo/1732329650667/Put/seqid=0 2024-11-23T02:42:03,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741869_1045 (size=5672) 2024-11-23T02:42:04,386 INFO [M:0;72b6db343a5d:43697 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/df634f57c20148ea9dd51a217e2d0b5e 2024-11-23T02:42:04,416 DEBUG [M:0;72b6db343a5d:43697 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96e54731f571416191b5c0197d93ba82 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732329654411/Put/seqid=0 2024-11-23T02:42:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741870_1046 (size=14067) 2024-11-23T02:42:04,821 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T02:42:04,823 INFO [M:0;72b6db343a5d:43697 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.40 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96e54731f571416191b5c0197d93ba82 2024-11-23T02:42:04,831 INFO [M:0;72b6db343a5d:43697 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 96e54731f571416191b5c0197d93ba82 2024-11-23T02:42:04,854 DEBUG [M:0;72b6db343a5d:43697 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1c9fdffb88b42b1a30d941d050e8ce5 is 69, key is 72b6db343a5d,43847,1732329616195/rs:state/1732329618505/Put/seqid=0 2024-11-23T02:42:04,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741871_1047 (size=5156) 2024-11-23T02:42:05,259 INFO [M:0;72b6db343a5d:43697 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1c9fdffb88b42b1a30d941d050e8ce5 2024-11-23T02:42:05,266 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/df634f57c20148ea9dd51a217e2d0b5e as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/df634f57c20148ea9dd51a217e2d0b5e 2024-11-23T02:42:05,272 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/df634f57c20148ea9dd51a217e2d0b5e, entries=8, sequenceid=317, filesize=5.5 K 2024-11-23T02:42:05,273 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96e54731f571416191b5c0197d93ba82 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96e54731f571416191b5c0197d93ba82 2024-11-23T02:42:05,279 INFO [M:0;72b6db343a5d:43697 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 96e54731f571416191b5c0197d93ba82 2024-11-23T02:42:05,279 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96e54731f571416191b5c0197d93ba82, entries=33, sequenceid=317, filesize=13.7 K 2024-11-23T02:42:05,280 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f1c9fdffb88b42b1a30d941d050e8ce5 as hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f1c9fdffb88b42b1a30d941d050e8ce5 2024-11-23T02:42:05,285 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46563/user/jenkins/test-data/b94000c3-2fd1-f797-c88c-e46f1f63ed3e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f1c9fdffb88b42b1a30d941d050e8ce5, entries=1, sequenceid=317, filesize=5.0 K 2024-11-23T02:42:05,286 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.96 KB/166867, heapSize ~190.53 KB/195104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1323ms, sequenceid=317, compaction requested=false 2024-11-23T02:42:05,288 INFO [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T02:42:05,288 DEBUG [M:0;72b6db343a5d:43697 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732329723962Disabling compacts and flushes for region at 1732329723962Disabling writes for close at 1732329723963 (+1 ms)Obtaining lock to block concurrent updates at 1732329723963Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732329723963Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166867, getHeapSize=195344, getOffHeapSize=0, getCellsCount=364 at 1732329723963Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732329723964 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732329723964Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732329723980 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732329723980Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732329724393 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732329724415 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732329724415Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732329724831 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732329724853 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732329724853Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77909a3f: reopening flushed file at 1732329725265 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53132f74: reopening flushed file at 1732329725272 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a51968e: reopening flushed file at 1732329725279 (+7 ms)Finished flush of dataSize ~162.96 KB/166867, heapSize ~190.53 KB/195104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1323ms, sequenceid=317, compaction requested=false at 1732329725286 (+7 ms)Writing region close event to WAL at 1732329725288 (+2 ms)Closed at 1732329725288 2024-11-23T02:42:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45173 is added to blk_1073741830_1006 (size=188079) 2024-11-23T02:42:05,291 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T02:42:05,291 INFO [M:0;72b6db343a5d:43697 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T02:42:05,291 INFO [M:0;72b6db343a5d:43697 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43697 2024-11-23T02:42:05,292 INFO [M:0;72b6db343a5d:43697 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T02:42:05,394 INFO [M:0;72b6db343a5d:43697 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T02:42:05,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T02:42:05,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43697-0x10142b217ae0000, quorum=127.0.0.1:64605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T02:42:05,398 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T02:42:05,400 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T02:42:05,401 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T02:42:05,401 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T02:42:05,401 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/hadoop.log.dir/,STOPPED} 2024-11-23T02:42:05,403 WARN [BP-1046908168-172.17.0.2-1732329612428 heartbeating to localhost/127.0.0.1:46563 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T02:42:05,403 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T02:42:05,403 WARN [BP-1046908168-172.17.0.2-1732329612428 heartbeating to localhost/127.0.0.1:46563 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1046908168-172.17.0.2-1732329612428 (Datanode Uuid ad12032d-151d-44fa-9769-8468792ac767) service to localhost/127.0.0.1:46563 2024-11-23T02:42:05,403 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T02:42:05,404 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/data/data1/current/BP-1046908168-172.17.0.2-1732329612428 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T02:42:05,405 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/cluster_62007a84-6b38-f2ad-86f7-a58eafee5f9e/data/data2/current/BP-1046908168-172.17.0.2-1732329612428 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T02:42:05,405 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T02:42:05,413 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T02:42:05,414 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T02:42:05,414 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T02:42:05,414 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T02:42:05,414 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4475b859-0954-0f78-0089-5509a63e0797/hadoop.log.dir/,STOPPED} 2024-11-23T02:42:05,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T02:42:05,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down