2024-11-28 08:36:24,683 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-28 08:36:24,703 main DEBUG Took 0.017873 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 08:36:24,704 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 08:36:24,704 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 08:36:24,705 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 08:36:24,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,716 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 08:36:24,733 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,736 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,737 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,739 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,741 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,742 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,744 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,745 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,746 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,747 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,748 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,750 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 08:36:24,751 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,751 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 08:36:24,753 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 08:36:24,755 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 08:36:24,757 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 08:36:24,758 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 08:36:24,760 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 08:36:24,760 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 08:36:24,771 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 08:36:24,774 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 08:36:24,776 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 08:36:24,777 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 08:36:24,777 main DEBUG createAppenders(={Console}) 2024-11-28 08:36:24,778 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-28 08:36:24,779 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-28 08:36:24,779 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-28 08:36:24,780 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 08:36:24,780 main DEBUG OutputStream closed 2024-11-28 08:36:24,781 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 08:36:24,781 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 08:36:24,781 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-28 08:36:24,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 08:36:24,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 08:36:24,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 08:36:24,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 08:36:24,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 08:36:24,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 08:36:24,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 08:36:24,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 08:36:24,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 08:36:24,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 08:36:24,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 08:36:24,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 08:36:24,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 08:36:24,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 08:36:24,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 08:36:24,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 08:36:24,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 08:36:24,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 08:36:24,877 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 08:36:24,878 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-28 08:36:24,878 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 08:36:24,878 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-28T08:36:24,893 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-28 08:36:24,896 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 08:36:24,897 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T08:36:25,149 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037 2024-11-28T08:36:25,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T08:36:25,188 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f, deleteOnExit=true 2024-11-28T08:36:25,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T08:36:25,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/test.cache.data in system properties and HBase conf 2024-11-28T08:36:25,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T08:36:25,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/hadoop.log.dir in system properties and HBase conf 2024-11-28T08:36:25,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T08:36:25,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T08:36:25,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T08:36:25,284 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T08:36:25,372 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T08:36:25,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T08:36:25,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T08:36:25,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T08:36:25,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T08:36:25,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T08:36:25,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T08:36:25,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T08:36:25,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T08:36:25,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T08:36:25,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/nfs.dump.dir in system properties and HBase conf 2024-11-28T08:36:25,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/java.io.tmpdir in system properties and HBase conf 2024-11-28T08:36:25,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T08:36:25,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T08:36:25,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T08:36:26,407 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T08:36:26,482 INFO [Time-limited test {}] log.Log(170): Logging initialized @2472ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T08:36:26,551 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T08:36:26,613 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T08:36:26,635 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T08:36:26,635 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T08:36:26,636 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T08:36:26,648 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T08:36:26,650 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/hadoop.log.dir/,AVAILABLE} 2024-11-28T08:36:26,651 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T08:36:26,828 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@562f0d0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/java.io.tmpdir/jetty-localhost-39487-hadoop-hdfs-3_4_1-tests_jar-_-any-10503081808470290694/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T08:36:26,839 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:39487} 2024-11-28T08:36:26,840 INFO [Time-limited test {}] server.Server(415): Started @2831ms 2024-11-28T08:36:27,470 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T08:36:27,478 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T08:36:27,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T08:36:27,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T08:36:27,480 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T08:36:27,480 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/hadoop.log.dir/,AVAILABLE} 2024-11-28T08:36:27,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T08:36:27,596 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc78d53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/java.io.tmpdir/jetty-localhost-36969-hadoop-hdfs-3_4_1-tests_jar-_-any-1804384798746952475/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T08:36:27,597 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:36969} 2024-11-28T08:36:27,597 INFO [Time-limited test {}] server.Server(415): Started @3588ms 2024-11-28T08:36:27,648 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T08:36:28,618 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/data/data1/current/BP-1694812817-172.17.0.3-1732782985901/current, will proceed with Du for space computation calculation, 2024-11-28T08:36:28,618 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/data/data2/current/BP-1694812817-172.17.0.3-1732782985901/current, will proceed with Du for space computation calculation, 2024-11-28T08:36:28,643 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T08:36:28,689 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11290ac7fa40743c with lease ID 0xa4d19993d76f1cbe: Processing first storage report for DS-5d90f506-3d6a-456c-a45e-94a5dfe45f86 from datanode DatanodeRegistration(127.0.0.1:38803, datanodeUuid=ed0374b9-ab4e-478d-8623-e28e1fc547d1, infoPort=44543, infoSecurePort=0, ipcPort=42585, storageInfo=lv=-57;cid=testClusterID;nsid=1779884001;c=1732782985901) 2024-11-28T08:36:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11290ac7fa40743c with lease ID 0xa4d19993d76f1cbe: from storage DS-5d90f506-3d6a-456c-a45e-94a5dfe45f86 node DatanodeRegistration(127.0.0.1:38803, datanodeUuid=ed0374b9-ab4e-478d-8623-e28e1fc547d1, infoPort=44543, infoSecurePort=0, ipcPort=42585, storageInfo=lv=-57;cid=testClusterID;nsid=1779884001;c=1732782985901), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-28T08:36:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11290ac7fa40743c with lease ID 0xa4d19993d76f1cbe: Processing first storage report for DS-4f194882-ef18-4198-9cdf-644f704859b7 from datanode DatanodeRegistration(127.0.0.1:38803, datanodeUuid=ed0374b9-ab4e-478d-8623-e28e1fc547d1, infoPort=44543, infoSecurePort=0, ipcPort=42585, storageInfo=lv=-57;cid=testClusterID;nsid=1779884001;c=1732782985901) 2024-11-28T08:36:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11290ac7fa40743c with lease ID 0xa4d19993d76f1cbe: from storage DS-4f194882-ef18-4198-9cdf-644f704859b7 node DatanodeRegistration(127.0.0.1:38803, datanodeUuid=ed0374b9-ab4e-478d-8623-e28e1fc547d1, infoPort=44543, infoSecurePort=0, ipcPort=42585, storageInfo=lv=-57;cid=testClusterID;nsid=1779884001;c=1732782985901), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T08:36:28,749 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037 2024-11-28T08:36:28,857 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/zookeeper_0, clientPort=57083, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T08:36:28,865 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57083 2024-11-28T08:36:28,880 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:28,884 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:29,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741825_1001 (size=7) 2024-11-28T08:36:29,503 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd with version=8 2024-11-28T08:36:29,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/hbase-staging 2024-11-28T08:36:29,579 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T08:36:29,815 INFO [Time-limited test {}] client.ConnectionUtils(128): master/81db972b9a1d:0 server-side Connection retries=9 2024-11-28T08:36:29,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T08:36:29,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T08:36:29,828 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T08:36:29,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T08:36:29,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T08:36:29,955 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T08:36:30,006 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T08:36:30,014 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T08:36:30,018 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T08:36:30,040 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 114178 (auto-detected) 2024-11-28T08:36:30,041 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-28T08:36:30,058 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36019 2024-11-28T08:36:30,084 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36019 connecting to ZooKeeper ensemble=127.0.0.1:57083 2024-11-28T08:36:30,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360190x0, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T08:36:30,237 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36019-0x101809619d40000 connected 2024-11-28T08:36:30,318 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:30,323 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:30,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T08:36:30,335 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd, hbase.cluster.distributed=false 2024-11-28T08:36:30,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T08:36:30,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36019 2024-11-28T08:36:30,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36019 2024-11-28T08:36:30,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36019 2024-11-28T08:36:30,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36019 2024-11-28T08:36:30,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36019 2024-11-28T08:36:30,456 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/81db972b9a1d:0 server-side Connection retries=9 2024-11-28T08:36:30,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T08:36:30,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T08:36:30,458 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T08:36:30,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T08:36:30,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T08:36:30,461 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T08:36:30,463 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T08:36:30,464 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32805 2024-11-28T08:36:30,465 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32805 connecting to ZooKeeper ensemble=127.0.0.1:57083 2024-11-28T08:36:30,466 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:30,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328050x0, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T08:36:30,484 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328050x0, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T08:36:30,484 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32805-0x101809619d40001 connected 2024-11-28T08:36:30,488 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T08:36:30,496 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T08:36:30,498 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T08:36:30,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T08:36:30,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32805 2024-11-28T08:36:30,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32805 2024-11-28T08:36:30,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32805 2024-11-28T08:36:30,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32805 2024-11-28T08:36:30,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32805 2024-11-28T08:36:30,526 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;81db972b9a1d:36019 2024-11-28T08:36:30,527 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/81db972b9a1d,36019,1732782989669 2024-11-28T08:36:30,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T08:36:30,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T08:36:30,538 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/81db972b9a1d,36019,1732782989669 2024-11-28T08:36:30,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:30,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T08:36:30,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:30,569 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T08:36:30,570 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/81db972b9a1d,36019,1732782989669 from backup master directory 2024-11-28T08:36:30,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T08:36:30,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/81db972b9a1d,36019,1732782989669 2024-11-28T08:36:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T08:36:30,579 WARN [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T08:36:30,579 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=81db972b9a1d,36019,1732782989669 2024-11-28T08:36:30,581 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T08:36:30,583 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T08:36:30,636 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/hbase.id] with ID: 971733f7-ae46-4067-9251-946c0ad347a3 2024-11-28T08:36:30,636 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/.tmp/hbase.id 2024-11-28T08:36:30,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741826_1002 (size=42) 2024-11-28T08:36:31,048 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/.tmp/hbase.id]:[hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/hbase.id] 2024-11-28T08:36:31,097 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T08:36:31,103 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T08:36:31,122 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-28T08:36:31,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:31,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741827_1003 (size=196) 2024-11-28T08:36:31,199 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:36:31,202 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T08:36:31,215 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T08:36:31,219 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T08:36:31,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741828_1004 (size=1189) 2024-11-28T08:36:31,670 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store 2024-11-28T08:36:31,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741829_1005 (size=34) 2024-11-28T08:36:32,099 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T08:36:32,104 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:36:32,106 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T08:36:32,106 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:36:32,106 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:36:32,108 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T08:36:32,109 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:36:32,109 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:36:32,110 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732782992105Disabling compacts and flushes for region at 1732782992105Disabling writes for close at 1732782992108 (+3 ms)Writing region close event to WAL at 1732782992109 (+1 ms)Closed at 1732782992109 2024-11-28T08:36:32,112 WARN [master/81db972b9a1d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/.initializing 2024-11-28T08:36:32,112 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/WALs/81db972b9a1d,36019,1732782989669 2024-11-28T08:36:32,120 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T08:36:32,133 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=81db972b9a1d%2C36019%2C1732782989669, suffix=, logDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/WALs/81db972b9a1d,36019,1732782989669, archiveDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/oldWALs, maxLogs=10 2024-11-28T08:36:32,155 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/WALs/81db972b9a1d,36019,1732782989669/81db972b9a1d%2C36019%2C1732782989669.1732782992138, exclude list is [], retry=0 2024-11-28T08:36:32,170 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38803,DS-5d90f506-3d6a-456c-a45e-94a5dfe45f86,DISK] 2024-11-28T08:36:32,173 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T08:36:32,204 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/WALs/81db972b9a1d,36019,1732782989669/81db972b9a1d%2C36019%2C1732782989669.1732782992138 2024-11-28T08:36:32,205 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44543:44543)] 2024-11-28T08:36:32,205 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:36:32,206 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:36:32,208 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,209 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T08:36:32,269 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:32,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:36:32,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T08:36:32,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:32,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:36:32,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T08:36:32,280 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:32,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:36:32,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T08:36:32,284 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:32,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:36:32,285 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,289 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,290 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,296 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,297 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,302 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T08:36:32,307 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T08:36:32,312 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:36:32,313 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63316906, jitterRate=-0.05650457739830017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T08:36:32,320 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732782992220Initializing all the Stores at 1732782992222 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732782992223 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732782992224 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732782992224Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732782992224Cleaning up temporary data from old regions at 1732782992297 (+73 ms)Region opened successfully at 1732782992320 (+23 ms) 2024-11-28T08:36:32,322 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T08:36:32,355 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5f7a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=81db972b9a1d/172.17.0.3:0 2024-11-28T08:36:32,388 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T08:36:32,399 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T08:36:32,399 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T08:36:32,401 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T08:36:32,403 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T08:36:32,408 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-28T08:36:32,408 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T08:36:32,431 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T08:36:32,439 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T08:36:32,483 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T08:36:32,487 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T08:36:32,490 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T08:36:32,503 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T08:36:32,506 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T08:36:32,512 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T08:36:32,524 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T08:36:32,526 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T08:36:32,536 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T08:36:32,558 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T08:36:32,567 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T08:36:32,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T08:36:32,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T08:36:32,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:32,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:32,582 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=81db972b9a1d,36019,1732782989669, sessionid=0x101809619d40000, setting cluster-up flag (Was=false) 2024-11-28T08:36:32,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:32,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:32,683 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T08:36:32,686 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=81db972b9a1d,36019,1732782989669 2024-11-28T08:36:32,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:32,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:36:32,736 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T08:36:32,740 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=81db972b9a1d,36019,1732782989669 2024-11-28T08:36:32,751 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T08:36:32,814 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(746): ClusterId : 971733f7-ae46-4067-9251-946c0ad347a3 2024-11-28T08:36:32,817 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T08:36:32,820 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T08:36:32,829 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T08:36:32,833 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T08:36:32,833 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T08:36:32,835 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T08:36:32,840 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 81db972b9a1d,36019,1732782989669 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T08:36:32,842 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T08:36:32,843 DEBUG [RS:0;81db972b9a1d:32805 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7882f07a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=81db972b9a1d/172.17.0.3:0 2024-11-28T08:36:32,846 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/81db972b9a1d:0, corePoolSize=5, maxPoolSize=5 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/81db972b9a1d:0, corePoolSize=5, maxPoolSize=5 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/81db972b9a1d:0, corePoolSize=5, maxPoolSize=5 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/81db972b9a1d:0, corePoolSize=5, maxPoolSize=5 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/81db972b9a1d:0, corePoolSize=10, maxPoolSize=10 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/81db972b9a1d:0, corePoolSize=2, maxPoolSize=2 2024-11-28T08:36:32,847 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:32,851 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732783022851 2024-11-28T08:36:32,852 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T08:36:32,853 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T08:36:32,853 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T08:36:32,854 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T08:36:32,857 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T08:36:32,857 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T08:36:32,858 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T08:36:32,858 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T08:36:32,859 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;81db972b9a1d:32805 2024-11-28T08:36:32,858 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:32,861 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:32,862 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T08:36:32,862 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T08:36:32,863 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T08:36:32,863 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T08:36:32,863 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T08:36:32,863 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T08:36:32,864 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T08:36:32,865 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T08:36:32,866 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(2659): reportForDuty to master=81db972b9a1d,36019,1732782989669 with port=32805, startcode=1732782990425 2024-11-28T08:36:32,866 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T08:36:32,868 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/81db972b9a1d:0:becomeActiveMaster-HFileCleaner.large.0-1732782992867,5,FailOnTimeoutGroup] 2024-11-28T08:36:32,870 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/81db972b9a1d:0:becomeActiveMaster-HFileCleaner.small.0-1732782992869,5,FailOnTimeoutGroup] 2024-11-28T08:36:32,870 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:32,870 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T08:36:32,871 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:32,872 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:32,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741831_1007 (size=1321) 2024-11-28T08:36:32,880 DEBUG [RS:0;81db972b9a1d:32805 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T08:36:32,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59849, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T08:36:32,918 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 81db972b9a1d,32805,1732782990425 2024-11-28T08:36:32,920 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36019 {}] master.ServerManager(517): Registering regionserver=81db972b9a1d,32805,1732782990425 2024-11-28T08:36:32,932 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:36:32,933 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33719 2024-11-28T08:36:32,933 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T08:36:32,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T08:36:32,947 DEBUG [RS:0;81db972b9a1d:32805 {}] zookeeper.ZKUtil(111): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/81db972b9a1d,32805,1732782990425 2024-11-28T08:36:32,947 WARN [RS:0;81db972b9a1d:32805 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T08:36:32,947 INFO [RS:0;81db972b9a1d:32805 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T08:36:32,947 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425 2024-11-28T08:36:32,949 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [81db972b9a1d,32805,1732782990425] 2024-11-28T08:36:32,973 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T08:36:32,989 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T08:36:32,994 INFO [RS:0;81db972b9a1d:32805 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T08:36:32,994 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:32,995 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T08:36:33,000 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T08:36:33,001 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,001 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,001 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,002 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,002 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,002 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,002 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/81db972b9a1d:0, corePoolSize=2, maxPoolSize=2 2024-11-28T08:36:33,002 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,002 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,003 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,003 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,003 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,003 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/81db972b9a1d:0, corePoolSize=1, maxPoolSize=1 2024-11-28T08:36:33,003 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/81db972b9a1d:0, corePoolSize=3, maxPoolSize=3 2024-11-28T08:36:33,003 DEBUG [RS:0;81db972b9a1d:32805 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/81db972b9a1d:0, corePoolSize=3, maxPoolSize=3 2024-11-28T08:36:33,004 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,004 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,004 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,004 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,005 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,005 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,32805,1732782990425-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T08:36:33,022 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T08:36:33,024 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,32805,1732782990425-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,024 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,024 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.Replication(171): 81db972b9a1d,32805,1732782990425 started 2024-11-28T08:36:33,041 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T08:36:33,041 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1482): Serving as 81db972b9a1d,32805,1732782990425, RpcServer on 81db972b9a1d/172.17.0.3:32805, sessionid=0x101809619d40001 2024-11-28T08:36:33,042 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T08:36:33,042 DEBUG [RS:0;81db972b9a1d:32805 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 81db972b9a1d,32805,1732782990425 2024-11-28T08:36:33,042 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '81db972b9a1d,32805,1732782990425' 2024-11-28T08:36:33,043 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T08:36:33,044 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T08:36:33,044 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T08:36:33,044 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T08:36:33,045 DEBUG [RS:0;81db972b9a1d:32805 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 81db972b9a1d,32805,1732782990425 2024-11-28T08:36:33,045 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '81db972b9a1d,32805,1732782990425' 2024-11-28T08:36:33,045 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T08:36:33,045 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T08:36:33,046 DEBUG [RS:0;81db972b9a1d:32805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T08:36:33,046 INFO [RS:0;81db972b9a1d:32805 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T08:36:33,046 INFO [RS:0;81db972b9a1d:32805 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T08:36:33,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T08:36:33,277 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:36:33,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741832_1008 (size=32) 2024-11-28T08:36:33,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:36:33,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T08:36:33,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T08:36:33,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:33,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:36:33,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T08:36:33,698 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T08:36:33,698 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:33,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:36:33,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T08:36:33,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T08:36:33,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:33,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:36:33,703 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T08:36:33,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T08:36:33,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:36:33,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:36:33,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T08:36:33,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740 2024-11-28T08:36:33,709 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740 2024-11-28T08:36:33,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T08:36:33,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T08:36:33,713 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T08:36:33,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T08:36:33,718 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:36:33,719 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62402773, jitterRate=-0.07012622058391571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T08:36:33,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732782993688Initializing all the Stores at 1732782993690 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732782993690Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732782993690Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732782993690Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732782993690Cleaning up temporary data from old regions at 1732782993712 (+22 ms)Region opened successfully at 1732782993721 (+9 ms) 2024-11-28T08:36:33,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T08:36:33,722 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T08:36:33,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T08:36:33,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T08:36:33,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T08:36:33,723 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T08:36:33,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732782993722Disabling compacts and flushes for region at 1732782993722Disabling writes for close at 1732782993722Writing region close event to WAL at 1732782993723 (+1 ms)Closed at 1732782993723 2024-11-28T08:36:33,726 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T08:36:33,726 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T08:36:33,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T08:36:33,739 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T08:36:33,742 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T08:36:33,894 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:34,895 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:35,897 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:36,898 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:37,899 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:38,901 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:39,902 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:40,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T08:36:40,005 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T08:36:40,903 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:41,904 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:42,905 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:43,906 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:44,907 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:45,908 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:46,909 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:47,910 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:48,911 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:49,912 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:50,914 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:51,915 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:52,916 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:53,917 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:54,919 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:55,920 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:56,921 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:57,922 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:58,744 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T08:36:58,923 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:36:59,925 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:37:00,926 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:37:01,927 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:37:02,928 WARN [81db972b9a1d:36019 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T08:37:03,058 INFO [RS:0;81db972b9a1d:32805 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T08:37:03,062 INFO [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=81db972b9a1d%2C32805%2C1732782990425, suffix=, logDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425, archiveDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/oldWALs, maxLogs=32 2024-11-28T08:37:03,077 DEBUG [RS:0;81db972b9a1d:32805 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425/81db972b9a1d%2C32805%2C1732782990425.1732783023064, exclude list is [], retry=0 2024-11-28T08:37:03,082 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38803,DS-5d90f506-3d6a-456c-a45e-94a5dfe45f86,DISK] 2024-11-28T08:37:03,085 INFO [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425/81db972b9a1d%2C32805%2C1732782990425.1732783023064 2024-11-28T08:37:03,085 DEBUG [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44543:44543)] 2024-11-28T08:37:03,182 DEBUG [81db972b9a1d:36019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T08:37:03,198 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:03,203 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 81db972b9a1d,32805,1732782990425, state=OPENING 2024-11-28T08:37:03,260 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T08:37:03,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:37:03,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:37:03,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T08:37:03,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T08:37:03,276 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T08:37:03,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:03,460 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T08:37:03,463 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57921, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T08:37:03,475 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T08:37:03,475 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T08:37:03,476 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T08:37:03,479 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=81db972b9a1d%2C32805%2C1732782990425.meta, suffix=.meta, logDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425, archiveDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/oldWALs, maxLogs=32 2024-11-28T08:37:03,493 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425/81db972b9a1d%2C32805%2C1732782990425.meta.1732783023481.meta, exclude list is [], retry=0 2024-11-28T08:37:03,497 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38803,DS-5d90f506-3d6a-456c-a45e-94a5dfe45f86,DISK] 2024-11-28T08:37:03,500 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425/81db972b9a1d%2C32805%2C1732782990425.meta.1732783023481.meta 2024-11-28T08:37:03,500 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44543:44543)] 2024-11-28T08:37:03,500 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:03,502 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T08:37:03,504 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T08:37:03,508 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T08:37:03,512 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T08:37:03,512 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:03,512 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T08:37:03,513 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T08:37:03,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T08:37:03,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T08:37:03,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:03,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:37:03,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T08:37:03,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T08:37:03,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:03,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:37:03,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T08:37:03,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T08:37:03,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:03,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:37:03,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T08:37:03,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T08:37:03,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:03,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T08:37:03,533 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T08:37:03,536 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740 2024-11-28T08:37:03,541 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740 2024-11-28T08:37:03,545 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T08:37:03,546 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T08:37:03,547 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T08:37:03,552 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T08:37:03,556 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66685266, jitterRate=-0.0063121020793914795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T08:37:03,557 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T08:37:03,559 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732783023513Writing region info on filesystem at 1732783023513Initializing all the Stores at 1732783023515 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732783023515Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732783023515Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783023515Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732783023515Cleaning up temporary data from old regions at 1732783023546 (+31 ms)Running coprocessor post-open hooks at 1732783023557 (+11 ms)Region opened successfully at 1732783023559 (+2 ms) 2024-11-28T08:37:03,572 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732783023451 2024-11-28T08:37:03,590 DEBUG [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T08:37:03,591 INFO [RS_OPEN_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T08:37:03,595 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:03,600 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 81db972b9a1d,32805,1732782990425, state=OPEN 2024-11-28T08:37:03,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T08:37:03,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T08:37:03,681 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T08:37:03,681 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T08:37:03,681 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:03,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T08:37:03,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=81db972b9a1d,32805,1732782990425 in 404 msec 2024-11-28T08:37:03,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T08:37:03,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.9590 sec 2024-11-28T08:37:03,699 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T08:37:03,699 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T08:37:03,718 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T08:37:03,720 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=81db972b9a1d,32805,1732782990425, seqNum=-1] 2024-11-28T08:37:03,741 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T08:37:03,744 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36223, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T08:37:03,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9880 sec 2024-11-28T08:37:03,768 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732783023768, completionTime=-1 2024-11-28T08:37:03,771 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T08:37:03,771 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T08:37:03,796 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T08:37:03,797 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732783083796 2024-11-28T08:37:03,797 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732783143797 2024-11-28T08:37:03,797 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-28T08:37:03,799 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,36019,1732782989669-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,799 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,36019,1732782989669-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,799 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,36019,1732782989669-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,800 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-81db972b9a1d:36019, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,801 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,801 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,808 DEBUG [master/81db972b9a1d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T08:37:03,829 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.250sec 2024-11-28T08:37:03,830 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T08:37:03,832 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T08:37:03,833 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T08:37:03,833 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T08:37:03,833 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T08:37:03,834 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,36019,1732782989669-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T08:37:03,835 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,36019,1732782989669-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T08:37:03,843 DEBUG [master/81db972b9a1d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T08:37:03,844 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T08:37:03,844 INFO [master/81db972b9a1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=81db972b9a1d,36019,1732782989669-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T08:37:03,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5e818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T08:37:03,867 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T08:37:03,867 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T08:37:03,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 81db972b9a1d,36019,-1 for getting cluster id 2024-11-28T08:37:03,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T08:37:03,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '971733f7-ae46-4067-9251-946c0ad347a3' 2024-11-28T08:37:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T08:37:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "971733f7-ae46-4067-9251-946c0ad347a3" 2024-11-28T08:37:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18214de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T08:37:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [81db972b9a1d,36019,-1] 2024-11-28T08:37:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T08:37:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:37:03,890 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T08:37:03,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ba9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T08:37:03,894 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T08:37:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=81db972b9a1d,32805,1732782990425, seqNum=-1] 2024-11-28T08:37:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T08:37:03,904 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T08:37:03,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=81db972b9a1d,36019,1732782989669 2024-11-28T08:37:03,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14688263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T08:37:03,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 81db972b9a1d,36019,-1 for getting cluster id 2024-11-28T08:37:03,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T08:37:03,927 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '971733f7-ae46-4067-9251-946c0ad347a3' 2024-11-28T08:37:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T08:37:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "971733f7-ae46-4067-9251-946c0ad347a3" 2024-11-28T08:37:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adcb0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T08:37:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [81db972b9a1d,36019,-1] 2024-11-28T08:37:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T08:37:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:37:03,930 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T08:37:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a188c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T08:37:03,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=6366 2024-11-28T08:37:03,950 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T08:37:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 81db972b9a1d,36019,1732782989669 2024-11-28T08:37:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@494ab692 2024-11-28T08:37:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T08:37:03,962 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T08:37:03,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-28T08:37:03,986 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:03,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-28T08:37:03,989 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:04,008 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T08:37:04,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741835_1011 (size=428) 2024-11-28T08:37:04,438 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a1413cdeef46e22d3ec726a59a54296b, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:04,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741836_1012 (size=79) 2024-11-28T08:37:04,854 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:04,854 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing a1413cdeef46e22d3ec726a59a54296b, disabling compactions & flushes 2024-11-28T08:37:04,855 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:04,855 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:04,855 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. after waiting 0 ms 2024-11-28T08:37:04,855 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:04,856 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:04,856 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a1413cdeef46e22d3ec726a59a54296b: Waiting for close lock at 1732783024854Disabling compacts and flushes for region at 1732783024854Disabling writes for close at 1732783024855 (+1 ms)Writing region close event to WAL at 1732783024855Closed at 1732783024855 2024-11-28T08:37:04,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:04,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732783024860"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783024860"}]},"ts":"1732783024860"} 2024-11-28T08:37:04,870 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:04,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:04,874 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783024872"}]},"ts":"1732783024872"} 2024-11-28T08:37:04,878 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-28T08:37:04,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a1413cdeef46e22d3ec726a59a54296b, ASSIGN}] 2024-11-28T08:37:04,883 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a1413cdeef46e22d3ec726a59a54296b, ASSIGN 2024-11-28T08:37:04,885 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a1413cdeef46e22d3ec726a59a54296b, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:05,037 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a1413cdeef46e22d3ec726a59a54296b, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T08:37:05,044 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a1413cdeef46e22d3ec726a59a54296b, ASSIGN because future has completed 2024-11-28T08:37:05,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1413cdeef46e22d3ec726a59a54296b, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:05,213 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:05,213 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a1413cdeef46e22d3ec726a59a54296b, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:05,214 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,215 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:05,215 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,215 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,218 INFO [StoreOpener-a1413cdeef46e22d3ec726a59a54296b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,221 INFO [StoreOpener-a1413cdeef46e22d3ec726a59a54296b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1413cdeef46e22d3ec726a59a54296b columnFamilyName testFamily 2024-11-28T08:37:05,221 DEBUG [StoreOpener-a1413cdeef46e22d3ec726a59a54296b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:05,222 INFO [StoreOpener-a1413cdeef46e22d3ec726a59a54296b-1 {}] regionserver.HStore(327): Store=a1413cdeef46e22d3ec726a59a54296b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:05,223 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,224 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,225 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,225 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,225 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,228 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,232 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:05,232 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a1413cdeef46e22d3ec726a59a54296b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69843098, jitterRate=0.04074326157569885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:05,233 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:37:05,234 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a1413cdeef46e22d3ec726a59a54296b: Running coprocessor pre-open hook at 1732783025215Writing region info on filesystem at 1732783025215Initializing all the Stores at 1732783025218 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783025218Cleaning up temporary data from old regions at 1732783025225 (+7 ms)Running coprocessor post-open hooks at 1732783025233 (+8 ms)Region opened successfully at 1732783025233 2024-11-28T08:37:05,235 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b., pid=6, masterSystemTime=1732783025202 2024-11-28T08:37:05,240 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:05,240 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:37:05,241 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a1413cdeef46e22d3ec726a59a54296b, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:05,246 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1413cdeef46e22d3ec726a59a54296b, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:05,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T08:37:05,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a1413cdeef46e22d3ec726a59a54296b, server=81db972b9a1d,32805,1732782990425 in 203 msec 2024-11-28T08:37:05,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T08:37:05,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a1413cdeef46e22d3ec726a59a54296b, ASSIGN in 372 msec 2024-11-28T08:37:05,258 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:05,259 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783025258"}]},"ts":"1732783025258"} 2024-11-28T08:37:05,262 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-28T08:37:05,263 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:05,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2860 sec 2024-11-28T08:37:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T08:37:07,061 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-28T08:37:07,061 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-28T08:37:07,063 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:07,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-28T08:37:07,077 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:07,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-28T08:37:07,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T08:37:07,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=81db972b9a1d,32805,1732782990425, seqNum=-1] 2024-11-28T08:37:07,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T08:37:07,087 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T08:37:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:07,108 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:07,111 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-28T08:37:07,117 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-28T08:37:07,121 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-28T08:37:07,123 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-28T08:37:07,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8643842573368033288 2024-11-28T08:37:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783030123, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:07,147 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-28T08:37:07,149 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-28T08:37:07,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:07,152 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:07,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8643842573368033290 2024-11-28T08:37:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783030152, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:07,157 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:07,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=8643842573368033291 2024-11-28T08:37:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783030157, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:07,162 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:07,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=8643842573368033292 2024-11-28T08:37:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783030161, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:07,166 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:07,177 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922654178_22 at /127.0.0.1:49032 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=6319 (was 6366) 2024-11-28T08:37:07,184 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=6318 2024-11-28T08:37:07,188 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:07,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-28T08:37:07,192 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:07,193 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:07,193 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-28T08:37:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T08:37:07,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:07,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741837_1013 (size=414) 2024-11-28T08:37:07,611 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a10d1d28856433fd41135169cd86b59, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:07,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741838_1014 (size=65) 2024-11-28T08:37:08,024 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:08,024 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 5a10d1d28856433fd41135169cd86b59, disabling compactions & flushes 2024-11-28T08:37:08,024 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,024 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,024 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. after waiting 0 ms 2024-11-28T08:37:08,024 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,024 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,025 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a10d1d28856433fd41135169cd86b59: Waiting for close lock at 1732783028024Disabling compacts and flushes for region at 1732783028024Disabling writes for close at 1732783028024Writing region close event to WAL at 1732783028024Closed at 1732783028024 2024-11-28T08:37:08,029 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:08,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732783028029"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783028029"}]},"ts":"1732783028029"} 2024-11-28T08:37:08,035 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:08,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:08,037 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783028036"}]},"ts":"1732783028036"} 2024-11-28T08:37:08,040 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-28T08:37:08,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5a10d1d28856433fd41135169cd86b59, ASSIGN}] 2024-11-28T08:37:08,042 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5a10d1d28856433fd41135169cd86b59, ASSIGN 2024-11-28T08:37:08,044 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5a10d1d28856433fd41135169cd86b59, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:08,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5a10d1d28856433fd41135169cd86b59, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:08,199 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5a10d1d28856433fd41135169cd86b59, ASSIGN because future has completed 2024-11-28T08:37:08,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a10d1d28856433fd41135169cd86b59, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T08:37:08,366 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a10d1d28856433fd41135169cd86b59, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:08,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:08,368 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,368 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,371 INFO [StoreOpener-5a10d1d28856433fd41135169cd86b59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,375 INFO [StoreOpener-5a10d1d28856433fd41135169cd86b59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a10d1d28856433fd41135169cd86b59 columnFamilyName testFamily 2024-11-28T08:37:08,375 DEBUG [StoreOpener-5a10d1d28856433fd41135169cd86b59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:08,376 INFO [StoreOpener-5a10d1d28856433fd41135169cd86b59-1 {}] regionserver.HStore(327): Store=5a10d1d28856433fd41135169cd86b59/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:08,376 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,384 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:08,385 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 5a10d1d28856433fd41135169cd86b59; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69674325, jitterRate=0.0382283478975296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:08,385 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:37:08,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5a10d1d28856433fd41135169cd86b59: Running coprocessor pre-open hook at 1732783028368Writing region info on filesystem at 1732783028368Initializing all the Stores at 1732783028370 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783028371 (+1 ms)Cleaning up temporary data from old regions at 1732783028379 (+8 ms)Running coprocessor post-open hooks at 1732783028385 (+6 ms)Region opened successfully at 1732783028386 (+1 ms) 2024-11-28T08:37:08,388 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., pid=9, masterSystemTime=1732783028354 2024-11-28T08:37:08,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,391 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:37:08,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5a10d1d28856433fd41135169cd86b59, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:08,396 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a10d1d28856433fd41135169cd86b59, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:08,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T08:37:08,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a10d1d28856433fd41135169cd86b59, server=81db972b9a1d,32805,1732782990425 in 198 msec 2024-11-28T08:37:08,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T08:37:08,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5a10d1d28856433fd41135169cd86b59, ASSIGN in 361 msec 2024-11-28T08:37:08,406 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:08,406 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783028406"}]},"ts":"1732783028406"} 2024-11-28T08:37:08,409 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-28T08:37:08,410 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:08,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2220 sec 2024-11-28T08:37:09,066 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:37:09,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-28T08:37:09,098 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T08:37:09,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-28T08:37:10,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T08:37:10,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:10,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T08:37:10,004 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T08:37:10,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-28T08:37:10,005 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:10,007 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-28T08:37:10,007 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T08:37:10,231 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-28T08:37:10,231 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-28T08:37:10,231 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:10,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-28T08:37:10,241 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:10,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-28T08:37:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:10,259 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:10,259 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-28T08:37:10,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T08:37:10,264 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-28T08:37:16,272 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., hostname=81db972b9a1d,32805,1732782990425, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=81db972b9a1d:32805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-28T08:37:16,273 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-28T08:37:16,273 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 from cache 2024-11-28T08:37:16,284 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=135 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=6304 (was 6318) 2024-11-28T08:37:16,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=135, ProcessCount=11, AvailableMemoryMB=6304 2024-11-28T08:37:16,295 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:16,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-28T08:37:16,300 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:16,300 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:16,300 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-28T08:37:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T08:37:16,303 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:16,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741839_1015 (size=412) 2024-11-28T08:37:16,716 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38932b92ba0f789ca2e99146b63e23c3, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741840_1016 (size=63) 2024-11-28T08:37:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-28T08:37:17,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:17,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 38932b92ba0f789ca2e99146b63e23c3, disabling compactions & flushes 2024-11-28T08:37:17,128 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. after waiting 0 ms 2024-11-28T08:37:17,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,128 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38932b92ba0f789ca2e99146b63e23c3: Waiting for close lock at 1732783037128Disabling compacts and flushes for region at 1732783037128Disabling writes for close at 1732783037128Writing region close event to WAL at 1732783037128Closed at 1732783037128 2024-11-28T08:37:17,130 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:17,131 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732783037130"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783037130"}]},"ts":"1732783037130"} 2024-11-28T08:37:17,134 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:17,135 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:17,136 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783037135"}]},"ts":"1732783037135"} 2024-11-28T08:37:17,138 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-28T08:37:17,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38932b92ba0f789ca2e99146b63e23c3, ASSIGN}] 2024-11-28T08:37:17,140 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38932b92ba0f789ca2e99146b63e23c3, ASSIGN 2024-11-28T08:37:17,142 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38932b92ba0f789ca2e99146b63e23c3, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:17,292 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=38932b92ba0f789ca2e99146b63e23c3, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:17,296 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38932b92ba0f789ca2e99146b63e23c3, ASSIGN because future has completed 2024-11-28T08:37:17,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38932b92ba0f789ca2e99146b63e23c3, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T08:37:17,466 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,466 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 38932b92ba0f789ca2e99146b63e23c3, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:17,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:17,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,470 INFO [StoreOpener-38932b92ba0f789ca2e99146b63e23c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,473 INFO [StoreOpener-38932b92ba0f789ca2e99146b63e23c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38932b92ba0f789ca2e99146b63e23c3 columnFamilyName testFamily 2024-11-28T08:37:17,473 DEBUG [StoreOpener-38932b92ba0f789ca2e99146b63e23c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:17,473 INFO [StoreOpener-38932b92ba0f789ca2e99146b63e23c3-1 {}] regionserver.HStore(327): Store=38932b92ba0f789ca2e99146b63e23c3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:17,474 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,475 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,479 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:17,484 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 38932b92ba0f789ca2e99146b63e23c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59169308, jitterRate=-0.11830860376358032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:17,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:37:17,485 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 38932b92ba0f789ca2e99146b63e23c3: Running coprocessor pre-open hook at 1732783037467Writing region info on filesystem at 1732783037467Initializing all the Stores at 1732783037469 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783037469Cleaning up temporary data from old regions at 1732783037476 (+7 ms)Running coprocessor post-open hooks at 1732783037484 (+8 ms)Region opened successfully at 1732783037484 2024-11-28T08:37:17,486 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3., pid=12, masterSystemTime=1732783037450 2024-11-28T08:37:17,489 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,489 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:37:17,490 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=38932b92ba0f789ca2e99146b63e23c3, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:17,493 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38932b92ba0f789ca2e99146b63e23c3, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:17,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T08:37:17,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 38932b92ba0f789ca2e99146b63e23c3, server=81db972b9a1d,32805,1732782990425 in 198 msec 2024-11-28T08:37:17,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-28T08:37:17,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38932b92ba0f789ca2e99146b63e23c3, ASSIGN in 361 msec 2024-11-28T08:37:17,503 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:17,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783037504"}]},"ts":"1732783037504"} 2024-11-28T08:37:17,507 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-28T08:37:17,508 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:17,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2130 sec 2024-11-28T08:37:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T08:37:19,352 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-28T08:37:19,352 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-28T08:37:19,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:19,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-28T08:37:19,366 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:19,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-28T08:37:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:19,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:19,383 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-28T08:37:19,387 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T08:37:19,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T08:37:19,392 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T08:37:19,393 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-28T08:37:20,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-28T08:37:20,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:21,103 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-28T08:37:23,002 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:37:25,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033301' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T08:37:25,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32805 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46266 deadline: 1732783051395, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033301' does not exist 2024-11-28T08:37:25,402 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 81db972b9a1d,32805,1732782990425 for closing scanner id = 8643842573368033301 for 38932b92ba0f789ca2e99146b63e23c3 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033301' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T08:37:25,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=152 (was 135) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7292 (was 6304) - AvailableMemoryMB LEAK? - 2024-11-28T08:37:25,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=152, ProcessCount=11, AvailableMemoryMB=7291 2024-11-28T08:37:25,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-28T08:37:25,419 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:25,420 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:25,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-28T08:37:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T08:37:25,421 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:25,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741841_1017 (size=423) 2024-11-28T08:37:25,834 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f6d20656c001a8c21b64a85595a60dcd, NAME => 'testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:25,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741842_1018 (size=74) 2024-11-28T08:37:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-28T08:37:26,250 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:26,250 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing f6d20656c001a8c21b64a85595a60dcd, disabling compactions & flushes 2024-11-28T08:37:26,250 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,250 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,250 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. after waiting 0 ms 2024-11-28T08:37:26,251 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,251 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,251 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for f6d20656c001a8c21b64a85595a60dcd: Waiting for close lock at 1732783046250Disabling compacts and flushes for region at 1732783046250Disabling writes for close at 1732783046250Writing region close event to WAL at 1732783046251 (+1 ms)Closed at 1732783046251 2024-11-28T08:37:26,255 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:26,256 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732783046255"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783046255"}]},"ts":"1732783046255"} 2024-11-28T08:37:26,261 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:26,263 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:26,264 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783046263"}]},"ts":"1732783046263"} 2024-11-28T08:37:26,268 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-28T08:37:26,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f6d20656c001a8c21b64a85595a60dcd, ASSIGN}] 2024-11-28T08:37:26,271 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f6d20656c001a8c21b64a85595a60dcd, ASSIGN 2024-11-28T08:37:26,273 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f6d20656c001a8c21b64a85595a60dcd, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:26,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f6d20656c001a8c21b64a85595a60dcd, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:26,427 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f6d20656c001a8c21b64a85595a60dcd, ASSIGN because future has completed 2024-11-28T08:37:26,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6d20656c001a8c21b64a85595a60dcd, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T08:37:26,585 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,585 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => f6d20656c001a8c21b64a85595a60dcd, NAME => 'testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:26,585 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,585 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:26,586 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,586 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,588 INFO [StoreOpener-f6d20656c001a8c21b64a85595a60dcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,591 INFO [StoreOpener-f6d20656c001a8c21b64a85595a60dcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6d20656c001a8c21b64a85595a60dcd columnFamilyName testFamily 2024-11-28T08:37:26,591 DEBUG [StoreOpener-f6d20656c001a8c21b64a85595a60dcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:26,592 INFO [StoreOpener-f6d20656c001a8c21b64a85595a60dcd-1 {}] regionserver.HStore(327): Store=f6d20656c001a8c21b64a85595a60dcd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:26,592 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,593 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,594 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,595 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,595 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,597 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,600 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:26,601 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened f6d20656c001a8c21b64a85595a60dcd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68204729, jitterRate=0.016329661011695862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:26,601 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:37:26,602 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for f6d20656c001a8c21b64a85595a60dcd: Running coprocessor pre-open hook at 1732783046586Writing region info on filesystem at 1732783046586Initializing all the Stores at 1732783046587 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783046587Cleaning up temporary data from old regions at 1732783046595 (+8 ms)Running coprocessor post-open hooks at 1732783046601 (+6 ms)Region opened successfully at 1732783046602 (+1 ms) 2024-11-28T08:37:26,603 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd., pid=15, masterSystemTime=1732783046580 2024-11-28T08:37:26,606 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,606 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:37:26,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f6d20656c001a8c21b64a85595a60dcd, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:26,611 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6d20656c001a8c21b64a85595a60dcd, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:26,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-28T08:37:26,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure f6d20656c001a8c21b64a85595a60dcd, server=81db972b9a1d,32805,1732782990425 in 185 msec 2024-11-28T08:37:26,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T08:37:26,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f6d20656c001a8c21b64a85595a60dcd, ASSIGN in 348 msec 2024-11-28T08:37:26,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:26,622 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783046622"}]},"ts":"1732783046622"} 2024-11-28T08:37:26,625 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-28T08:37:26,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:26,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2110 sec 2024-11-28T08:37:27,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-28T08:37:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T08:37:28,461 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-28T08:37:28,461 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-28T08:37:28,462 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:28,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-28T08:37:28,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:28,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-28T08:37:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:28,490 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:28,490 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-28T08:37:28,493 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-28T08:37:28,495 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-28T08:37:28,496 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-28T08:37:28,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8643842573368033305 2024-11-28T08:37:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783051495, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:28,500 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-28T08:37:28,502 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-28T08:37:28,502 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:28,504 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:28,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8643842573368033307 2024-11-28T08:37:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783051503, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:28,508 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:28,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=8643842573368033308 2024-11-28T08:37:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783051508, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:28,512 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:28,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=8643842573368033309 2024-11-28T08:37:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46266 deadline: 1732783051512, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T08:37:28,516 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T08:37:28,524 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922654178_22 at /127.0.0.1:52284 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=152 (was 152), ProcessCount=11 (was 11), AvailableMemoryMB=7156 (was 7291) 2024-11-28T08:37:28,533 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=152, ProcessCount=11, AvailableMemoryMB=7155 2024-11-28T08:37:28,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-28T08:37:28,539 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:28,539 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:28,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-28T08:37:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T08:37:28,541 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:28,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741843_1019 (size=418) 2024-11-28T08:37:28,745 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T08:37:28,957 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d4af0876fb90af949d407490d72c4923, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:28,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741844_1020 (size=69) 2024-11-28T08:37:29,373 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:29,373 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d4af0876fb90af949d407490d72c4923, disabling compactions & flushes 2024-11-28T08:37:29,373 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,374 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,374 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. after waiting 0 ms 2024-11-28T08:37:29,374 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,374 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,374 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d4af0876fb90af949d407490d72c4923: Waiting for close lock at 1732783049373Disabling compacts and flushes for region at 1732783049373Disabling writes for close at 1732783049374 (+1 ms)Writing region close event to WAL at 1732783049374Closed at 1732783049374 2024-11-28T08:37:29,376 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:29,377 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732783049376"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783049376"}]},"ts":"1732783049376"} 2024-11-28T08:37:29,381 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:29,383 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:29,383 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783049383"}]},"ts":"1732783049383"} 2024-11-28T08:37:29,386 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-28T08:37:29,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d4af0876fb90af949d407490d72c4923, ASSIGN}] 2024-11-28T08:37:29,389 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d4af0876fb90af949d407490d72c4923, ASSIGN 2024-11-28T08:37:29,390 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d4af0876fb90af949d407490d72c4923, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:29,541 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d4af0876fb90af949d407490d72c4923, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:29,544 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d4af0876fb90af949d407490d72c4923, ASSIGN because future has completed 2024-11-28T08:37:29,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4af0876fb90af949d407490d72c4923, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T08:37:29,704 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,704 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => d4af0876fb90af949d407490d72c4923, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:29,705 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,705 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:29,705 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,705 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,707 INFO [StoreOpener-d4af0876fb90af949d407490d72c4923-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,710 INFO [StoreOpener-d4af0876fb90af949d407490d72c4923-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4af0876fb90af949d407490d72c4923 columnFamilyName testFamily 2024-11-28T08:37:29,710 DEBUG [StoreOpener-d4af0876fb90af949d407490d72c4923-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:29,710 INFO [StoreOpener-d4af0876fb90af949d407490d72c4923-1 {}] regionserver.HStore(327): Store=d4af0876fb90af949d407490d72c4923/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:29,711 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,712 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,712 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,713 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,713 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,715 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,718 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:29,719 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened d4af0876fb90af949d407490d72c4923; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73872680, jitterRate=0.10078871250152588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:29,719 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4af0876fb90af949d407490d72c4923 2024-11-28T08:37:29,719 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for d4af0876fb90af949d407490d72c4923: Running coprocessor pre-open hook at 1732783049705Writing region info on filesystem at 1732783049705Initializing all the Stores at 1732783049707 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783049707Cleaning up temporary data from old regions at 1732783049713 (+6 ms)Running coprocessor post-open hooks at 1732783049719 (+6 ms)Region opened successfully at 1732783049719 2024-11-28T08:37:29,721 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., pid=18, masterSystemTime=1732783049698 2024-11-28T08:37:29,723 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,723 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:37:29,724 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d4af0876fb90af949d407490d72c4923, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:29,727 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4af0876fb90af949d407490d72c4923, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:29,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T08:37:29,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d4af0876fb90af949d407490d72c4923, server=81db972b9a1d,32805,1732782990425 in 183 msec 2024-11-28T08:37:29,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-28T08:37:29,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d4af0876fb90af949d407490d72c4923, ASSIGN in 345 msec 2024-11-28T08:37:29,738 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:29,738 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783049738"}]},"ts":"1732783049738"} 2024-11-28T08:37:29,741 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-28T08:37:29,743 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:29,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2080 sec 2024-11-28T08:37:30,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-28T08:37:30,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:30,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-28T08:37:30,003 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T08:37:31,568 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-28T08:37:31,569 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-28T08:37:31,569 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:31,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-28T08:37:31,574 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:31,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-28T08:37:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:31,591 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:31,591 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-28T08:37:31,594 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T08:37:31,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T08:37:32,088 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:37:32,604 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=81db972b9a1d:32805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-28T08:37:32,604 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-28T08:37:32,605 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 from cache 2024-11-28T08:37:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-28T08:37:33,105 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-28T08:37:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:33,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T08:37:34,605 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=81db972b9a1d:32805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=977ms 2024-11-28T08:37:34,605 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=977ms 2024-11-28T08:37:34,605 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 from cache 2024-11-28T08:37:34,617 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=160 (was 152) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6984 (was 7155) 2024-11-28T08:37:34,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=160, ProcessCount=11, AvailableMemoryMB=6984 2024-11-28T08:37:34,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-28T08:37:34,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:34,635 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:34,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-28T08:37:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T08:37:34,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:34,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741845_1021 (size=416) 2024-11-28T08:37:35,051 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a1d5c220899b33f9a0e277860d1eff68, NAME => 'testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:35,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741846_1022 (size=67) 2024-11-28T08:37:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-28T08:37:35,466 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:35,466 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing a1d5c220899b33f9a0e277860d1eff68, disabling compactions & flushes 2024-11-28T08:37:35,466 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,467 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,467 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. after waiting 0 ms 2024-11-28T08:37:35,467 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,467 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,467 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a1d5c220899b33f9a0e277860d1eff68: Waiting for close lock at 1732783055466Disabling compacts and flushes for region at 1732783055466Disabling writes for close at 1732783055467 (+1 ms)Writing region close event to WAL at 1732783055467Closed at 1732783055467 2024-11-28T08:37:35,469 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:35,469 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732783055469"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783055469"}]},"ts":"1732783055469"} 2024-11-28T08:37:35,472 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:35,474 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:35,474 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783055474"}]},"ts":"1732783055474"} 2024-11-28T08:37:35,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-28T08:37:35,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a1d5c220899b33f9a0e277860d1eff68, ASSIGN}] 2024-11-28T08:37:35,479 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a1d5c220899b33f9a0e277860d1eff68, ASSIGN 2024-11-28T08:37:35,481 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a1d5c220899b33f9a0e277860d1eff68, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:35,632 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=a1d5c220899b33f9a0e277860d1eff68, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:35,638 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a1d5c220899b33f9a0e277860d1eff68, ASSIGN because future has completed 2024-11-28T08:37:35,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1d5c220899b33f9a0e277860d1eff68, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T08:37:35,798 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,798 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => a1d5c220899b33f9a0e277860d1eff68, NAME => 'testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:35,799 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,799 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:35,799 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,799 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,801 INFO [StoreOpener-a1d5c220899b33f9a0e277860d1eff68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,803 INFO [StoreOpener-a1d5c220899b33f9a0e277860d1eff68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1d5c220899b33f9a0e277860d1eff68 columnFamilyName testFamily 2024-11-28T08:37:35,803 DEBUG [StoreOpener-a1d5c220899b33f9a0e277860d1eff68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:35,804 INFO [StoreOpener-a1d5c220899b33f9a0e277860d1eff68-1 {}] regionserver.HStore(327): Store=a1d5c220899b33f9a0e277860d1eff68/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:35,804 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,805 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,805 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,806 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,806 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,809 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,812 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:35,813 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened a1d5c220899b33f9a0e277860d1eff68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62584898, jitterRate=-0.0674123466014862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:35,813 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:37:35,814 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for a1d5c220899b33f9a0e277860d1eff68: Running coprocessor pre-open hook at 1732783055799Writing region info on filesystem at 1732783055799Initializing all the Stores at 1732783055801 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783055801Cleaning up temporary data from old regions at 1732783055806 (+5 ms)Running coprocessor post-open hooks at 1732783055813 (+7 ms)Region opened successfully at 1732783055814 (+1 ms) 2024-11-28T08:37:35,815 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68., pid=21, masterSystemTime=1732783055792 2024-11-28T08:37:35,818 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,818 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:37:35,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=a1d5c220899b33f9a0e277860d1eff68, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:35,822 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1d5c220899b33f9a0e277860d1eff68, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:35,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-28T08:37:35,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure a1d5c220899b33f9a0e277860d1eff68, server=81db972b9a1d,32805,1732782990425 in 184 msec 2024-11-28T08:37:35,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T08:37:35,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=a1d5c220899b33f9a0e277860d1eff68, ASSIGN in 349 msec 2024-11-28T08:37:35,830 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:35,830 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783055830"}]},"ts":"1732783055830"} 2024-11-28T08:37:35,833 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-28T08:37:35,835 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:35,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2040 sec 2024-11-28T08:37:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T08:37:37,669 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-28T08:37:37,669 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-28T08:37:37,670 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:37,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-28T08:37:37,678 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:37,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-28T08:37:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:37,692 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:37,693 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-28T08:37:37,697 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T08:37:37,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T08:37:37,702 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T08:37:37,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-28T08:37:39,108 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-28T08:37:40,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-28T08:37:40,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:40,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033320' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T08:37:40,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32805 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46266 deadline: 1732783061705, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033320' does not exist 2024-11-28T08:37:40,709 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 81db972b9a1d,32805,1732782990425 for closing scanner id = 8643842573368033320 for a1d5c220899b33f9a0e277860d1eff68 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033320' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T08:37:40,716 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=147 (was 160), ProcessCount=11 (was 11), AvailableMemoryMB=6945 (was 6984) 2024-11-28T08:37:40,726 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=147, ProcessCount=11, AvailableMemoryMB=6944 2024-11-28T08:37:40,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-28T08:37:40,733 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:40,733 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:40,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-28T08:37:40,735 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T08:37:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741847_1023 (size=411) 2024-11-28T08:37:41,150 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e24f2da55cdb9e86fc742191c3a6234d, NAME => 'testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:41,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741848_1024 (size=62) 2024-11-28T08:37:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-28T08:37:41,302 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:37:41,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:41,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e24f2da55cdb9e86fc742191c3a6234d, disabling compactions & flushes 2024-11-28T08:37:41,566 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,566 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,567 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. after waiting 0 ms 2024-11-28T08:37:41,567 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,567 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,567 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e24f2da55cdb9e86fc742191c3a6234d: Waiting for close lock at 1732783061566Disabling compacts and flushes for region at 1732783061566Disabling writes for close at 1732783061567 (+1 ms)Writing region close event to WAL at 1732783061567Closed at 1732783061567 2024-11-28T08:37:41,569 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:41,569 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732783061569"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783061569"}]},"ts":"1732783061569"} 2024-11-28T08:37:41,571 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:41,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:41,573 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783061573"}]},"ts":"1732783061573"} 2024-11-28T08:37:41,575 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-28T08:37:41,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e24f2da55cdb9e86fc742191c3a6234d, ASSIGN}] 2024-11-28T08:37:41,577 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e24f2da55cdb9e86fc742191c3a6234d, ASSIGN 2024-11-28T08:37:41,578 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e24f2da55cdb9e86fc742191c3a6234d, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:41,729 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e24f2da55cdb9e86fc742191c3a6234d, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:41,733 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e24f2da55cdb9e86fc742191c3a6234d, ASSIGN because future has completed 2024-11-28T08:37:41,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e24f2da55cdb9e86fc742191c3a6234d, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T08:37:41,898 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,898 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => e24f2da55cdb9e86fc742191c3a6234d, NAME => 'testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:41,899 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,899 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:41,899 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,899 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,902 INFO [StoreOpener-e24f2da55cdb9e86fc742191c3a6234d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,905 INFO [StoreOpener-e24f2da55cdb9e86fc742191c3a6234d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24f2da55cdb9e86fc742191c3a6234d columnFamilyName testFamily 2024-11-28T08:37:41,905 DEBUG [StoreOpener-e24f2da55cdb9e86fc742191c3a6234d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:41,906 INFO [StoreOpener-e24f2da55cdb9e86fc742191c3a6234d-1 {}] regionserver.HStore(327): Store=e24f2da55cdb9e86fc742191c3a6234d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:41,906 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,907 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,908 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,909 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,909 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,912 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,916 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:41,916 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened e24f2da55cdb9e86fc742191c3a6234d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64627424, jitterRate=-0.03697633743286133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:41,917 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:37:41,917 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for e24f2da55cdb9e86fc742191c3a6234d: Running coprocessor pre-open hook at 1732783061900Writing region info on filesystem at 1732783061900Initializing all the Stores at 1732783061901 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783061902 (+1 ms)Cleaning up temporary data from old regions at 1732783061909 (+7 ms)Running coprocessor post-open hooks at 1732783061917 (+8 ms)Region opened successfully at 1732783061917 2024-11-28T08:37:41,918 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d., pid=24, masterSystemTime=1732783061887 2024-11-28T08:37:41,921 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,921 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:37:41,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e24f2da55cdb9e86fc742191c3a6234d, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:41,924 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e24f2da55cdb9e86fc742191c3a6234d, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:41,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T08:37:41,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e24f2da55cdb9e86fc742191c3a6234d, server=81db972b9a1d,32805,1732782990425 in 192 msec 2024-11-28T08:37:41,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-28T08:37:41,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e24f2da55cdb9e86fc742191c3a6234d, ASSIGN in 351 msec 2024-11-28T08:37:41,931 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:41,931 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783061931"}]},"ts":"1732783061931"} 2024-11-28T08:37:41,933 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-28T08:37:41,935 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:41,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2060 sec 2024-11-28T08:37:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T08:37:43,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-28T08:37:43,762 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-28T08:37:43,763 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:43,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-28T08:37:43,774 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:43,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-28T08:37:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:43,789 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:43,789 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-28T08:37:43,792 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T08:37:43,794 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T08:37:43,795 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T08:37:43,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-28T08:37:44,049 INFO [master/81db972b9a1d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T08:37:44,049 INFO [master/81db972b9a1d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T08:37:45,112 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-28T08:37:46,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033324' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T08:37:46,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32805 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46266 deadline: 1732783067806, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033324' does not exist 2024-11-28T08:37:46,815 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 81db972b9a1d,32805,1732782990425 for closing scanner id = 8643842573368033324 for e24f2da55cdb9e86fc742191c3a6234d of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033324' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T08:37:46,824 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=135 (was 147), ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6944) 2024-11-28T08:37:46,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=135, ProcessCount=11, AvailableMemoryMB=6876 2024-11-28T08:37:46,835 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:46,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T08:37:46,839 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:46,839 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:46,839 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-28T08:37:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T08:37:46,841 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741849_1025 (size=419) 2024-11-28T08:37:47,253 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0ff3c4719c548dd9b9b6a0fdddf645c, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:47,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741850_1026 (size=70) 2024-11-28T08:37:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-28T08:37:47,402 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:37:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:47,663 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing d0ff3c4719c548dd9b9b6a0fdddf645c, disabling compactions & flushes 2024-11-28T08:37:47,663 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:47,664 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:47,664 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. after waiting 0 ms 2024-11-28T08:37:47,664 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:47,664 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:47,664 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0ff3c4719c548dd9b9b6a0fdddf645c: Waiting for close lock at 1732783067663Disabling compacts and flushes for region at 1732783067663Disabling writes for close at 1732783067664 (+1 ms)Writing region close event to WAL at 1732783067664Closed at 1732783067664 2024-11-28T08:37:47,666 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:47,667 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732783067666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783067666"}]},"ts":"1732783067666"} 2024-11-28T08:37:47,670 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:47,672 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:47,672 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783067672"}]},"ts":"1732783067672"} 2024-11-28T08:37:47,675 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-28T08:37:47,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d0ff3c4719c548dd9b9b6a0fdddf645c, ASSIGN}] 2024-11-28T08:37:47,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d0ff3c4719c548dd9b9b6a0fdddf645c, ASSIGN 2024-11-28T08:37:47,678 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d0ff3c4719c548dd9b9b6a0fdddf645c, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:47,829 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d0ff3c4719c548dd9b9b6a0fdddf645c, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:47,832 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d0ff3c4719c548dd9b9b6a0fdddf645c, ASSIGN because future has completed 2024-11-28T08:37:47,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0ff3c4719c548dd9b9b6a0fdddf645c, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T08:37:47,991 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:47,991 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => d0ff3c4719c548dd9b9b6a0fdddf645c, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:47,991 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:47,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:47,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:47,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:47,995 INFO [StoreOpener-d0ff3c4719c548dd9b9b6a0fdddf645c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:47,998 INFO [StoreOpener-d0ff3c4719c548dd9b9b6a0fdddf645c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0ff3c4719c548dd9b9b6a0fdddf645c columnFamilyName testFamily 2024-11-28T08:37:47,998 DEBUG [StoreOpener-d0ff3c4719c548dd9b9b6a0fdddf645c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:47,999 INFO [StoreOpener-d0ff3c4719c548dd9b9b6a0fdddf645c-1 {}] regionserver.HStore(327): Store=d0ff3c4719c548dd9b9b6a0fdddf645c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:47,999 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,005 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:48,010 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened d0ff3c4719c548dd9b9b6a0fdddf645c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74856218, jitterRate=0.11544457077980042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:48,010 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:37:48,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for d0ff3c4719c548dd9b9b6a0fdddf645c: Running coprocessor pre-open hook at 1732783067992Writing region info on filesystem at 1732783067992Initializing all the Stores at 1732783067993 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783067994 (+1 ms)Cleaning up temporary data from old regions at 1732783068003 (+9 ms)Running coprocessor post-open hooks at 1732783068010 (+7 ms)Region opened successfully at 1732783068011 (+1 ms) 2024-11-28T08:37:48,012 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., pid=27, masterSystemTime=1732783067985 2024-11-28T08:37:48,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:48,015 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:37:48,015 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d0ff3c4719c548dd9b9b6a0fdddf645c, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:48,018 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0ff3c4719c548dd9b9b6a0fdddf645c, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:48,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-28T08:37:48,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure d0ff3c4719c548dd9b9b6a0fdddf645c, server=81db972b9a1d,32805,1732782990425 in 186 msec 2024-11-28T08:37:48,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T08:37:48,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d0ff3c4719c548dd9b9b6a0fdddf645c, ASSIGN in 347 msec 2024-11-28T08:37:48,027 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:48,027 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783068027"}]},"ts":"1732783068027"} 2024-11-28T08:37:48,030 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-28T08:37:48,032 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:48,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1970 sec 2024-11-28T08:37:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T08:37:49,870 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-28T08:37:49,870 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-28T08:37:49,871 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:49,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-28T08:37:49,884 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:49,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-28T08:37:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:49,896 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:49,897 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-28T08:37:49,900 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T08:37:49,901 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-28T08:37:50,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-28T08:37:50,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:50,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T08:37:50,003 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T08:37:51,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-28T08:37:53,494 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:37:55,905 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., hostname=81db972b9a1d,32805,1732782990425, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=81db972b9a1d:32805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-11-28T08:37:55,906 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-11-28T08:37:55,906 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 from cache 2024-11-28T08:37:55,924 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 135), ProcessCount=11 (was 11), AvailableMemoryMB=6719 (was 6876) 2024-11-28T08:37:55,931 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=6718 2024-11-28T08:37:55,933 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:37:55,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-28T08:37:55,936 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:37:55,936 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:55,936 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-28T08:37:55,938 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:37:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T08:37:55,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741851_1027 (size=407) 2024-11-28T08:37:56,354 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46db51c6773fee2566ff7a311e132c68, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:37:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-28T08:37:56,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741852_1028 (size=58) 2024-11-28T08:37:56,846 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:56,846 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 46db51c6773fee2566ff7a311e132c68, disabling compactions & flushes 2024-11-28T08:37:56,846 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:56,846 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:56,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. after waiting 0 ms 2024-11-28T08:37:56,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:56,847 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:56,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46db51c6773fee2566ff7a311e132c68: Waiting for close lock at 1732783076846Disabling compacts and flushes for region at 1732783076846Disabling writes for close at 1732783076847 (+1 ms)Writing region close event to WAL at 1732783076847Closed at 1732783076847 2024-11-28T08:37:56,850 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:37:56,851 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732783076850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783076850"}]},"ts":"1732783076850"} 2024-11-28T08:37:56,855 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:37:56,857 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:37:56,858 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783076857"}]},"ts":"1732783076857"} 2024-11-28T08:37:56,861 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-28T08:37:56,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=46db51c6773fee2566ff7a311e132c68, ASSIGN}] 2024-11-28T08:37:56,864 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=46db51c6773fee2566ff7a311e132c68, ASSIGN 2024-11-28T08:37:56,866 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=46db51c6773fee2566ff7a311e132c68, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:37:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T08:37:57,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=46db51c6773fee2566ff7a311e132c68, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:57,021 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=46db51c6773fee2566ff7a311e132c68, ASSIGN because future has completed 2024-11-28T08:37:57,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46db51c6773fee2566ff7a311e132c68, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:37:57,187 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:57,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 46db51c6773fee2566ff7a311e132c68, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:37:57,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:37:57,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,190 INFO [StoreOpener-46db51c6773fee2566ff7a311e132c68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,193 INFO [StoreOpener-46db51c6773fee2566ff7a311e132c68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46db51c6773fee2566ff7a311e132c68 columnFamilyName testFamily 2024-11-28T08:37:57,193 DEBUG [StoreOpener-46db51c6773fee2566ff7a311e132c68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:37:57,193 INFO [StoreOpener-46db51c6773fee2566ff7a311e132c68-1 {}] regionserver.HStore(327): Store=46db51c6773fee2566ff7a311e132c68/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:37:57,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,196 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,196 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,199 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,203 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:37:57,204 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 46db51c6773fee2566ff7a311e132c68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73640486, jitterRate=0.09732875227928162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:37:57,204 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:37:57,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 46db51c6773fee2566ff7a311e132c68: Running coprocessor pre-open hook at 1732783077188Writing region info on filesystem at 1732783077188Initializing all the Stores at 1732783077190 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783077190Cleaning up temporary data from old regions at 1732783077196 (+6 ms)Running coprocessor post-open hooks at 1732783077204 (+8 ms)Region opened successfully at 1732783077205 (+1 ms) 2024-11-28T08:37:57,206 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68., pid=30, masterSystemTime=1732783077176 2024-11-28T08:37:57,209 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:57,209 INFO [RS_OPEN_PRIORITY_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:37:57,210 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=46db51c6773fee2566ff7a311e132c68, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:37:57,213 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46db51c6773fee2566ff7a311e132c68, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:37:57,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T08:37:57,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 46db51c6773fee2566ff7a311e132c68, server=81db972b9a1d,32805,1732782990425 in 193 msec 2024-11-28T08:37:57,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-28T08:37:57,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=46db51c6773fee2566ff7a311e132c68, ASSIGN in 355 msec 2024-11-28T08:37:57,222 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:37:57,222 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783077222"}]},"ts":"1732783077222"} 2024-11-28T08:37:57,225 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-28T08:37:57,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:37:57,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2930 sec 2024-11-28T08:37:58,745 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T08:37:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T08:37:58,961 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-28T08:37:58,961 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-28T08:37:58,962 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:58,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-28T08:37:58,971 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:37:58,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-28T08:37:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:37:58,985 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:37:58,985 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-28T08:37:58,989 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T08:37:58,991 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T08:37:58,993 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T08:37:58,994 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-28T08:38:00,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-28T08:38:00,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-28T08:38:02,694 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:38:03,117 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-28T08:38:04,325 DEBUG [master/81db972b9a1d:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T08:38:05,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033332' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T08:38:05,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32805 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46266 deadline: 1732783090997, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033332' does not exist 2024-11-28T08:38:05,009 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 81db972b9a1d,32805,1732782990425 for closing scanner id = 8643842573368033332 for 46db51c6773fee2566ff7a311e132c68 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8643842573368033332' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T08:38:05,014 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=6591 (was 6718) 2024-11-28T08:38:05,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=6590 2024-11-28T08:38:05,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T08:38:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-28T08:38:05,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T08:38:05,029 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:38:05,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-28T08:38:05,030 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T08:38:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T08:38:05,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741853_1029 (size=423) 2024-11-28T08:38:05,446 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a54460b3855674c4d88c6c0a82684355, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd 2024-11-28T08:38:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741854_1030 (size=74) 2024-11-28T08:38:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-28T08:38:05,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:38:05,865 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing a54460b3855674c4d88c6c0a82684355, disabling compactions & flushes 2024-11-28T08:38:05,865 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:05,865 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:05,865 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. after waiting 0 ms 2024-11-28T08:38:05,865 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:05,865 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:05,866 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a54460b3855674c4d88c6c0a82684355: Waiting for close lock at 1732783085864Disabling compacts and flushes for region at 1732783085864Disabling writes for close at 1732783085865 (+1 ms)Writing region close event to WAL at 1732783085865Closed at 1732783085865 2024-11-28T08:38:05,870 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T08:38:05,871 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732783085870"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732783085870"}]},"ts":"1732783085870"} 2024-11-28T08:38:05,875 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T08:38:05,876 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T08:38:05,877 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783085876"}]},"ts":"1732783085876"} 2024-11-28T08:38:05,880 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-28T08:38:05,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a54460b3855674c4d88c6c0a82684355, ASSIGN}] 2024-11-28T08:38:05,882 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a54460b3855674c4d88c6c0a82684355, ASSIGN 2024-11-28T08:38:05,883 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a54460b3855674c4d88c6c0a82684355, ASSIGN; state=OFFLINE, location=81db972b9a1d,32805,1732782990425; forceNewPlan=false, retain=false 2024-11-28T08:38:06,034 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=a54460b3855674c4d88c6c0a82684355, regionState=OPENING, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:38:06,038 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a54460b3855674c4d88c6c0a82684355, ASSIGN because future has completed 2024-11-28T08:38:06,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure a54460b3855674c4d88c6c0a82684355, server=81db972b9a1d,32805,1732782990425}] 2024-11-28T08:38:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T08:38:06,205 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:06,206 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => a54460b3855674c4d88c6c0a82684355, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355.', STARTKEY => '', ENDKEY => ''} 2024-11-28T08:38:06,207 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,207 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T08:38:06,207 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,208 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,210 INFO [StoreOpener-a54460b3855674c4d88c6c0a82684355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,213 INFO [StoreOpener-a54460b3855674c4d88c6c0a82684355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a54460b3855674c4d88c6c0a82684355 columnFamilyName testFamily 2024-11-28T08:38:06,213 DEBUG [StoreOpener-a54460b3855674c4d88c6c0a82684355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T08:38:06,214 INFO [StoreOpener-a54460b3855674c4d88c6c0a82684355-1 {}] regionserver.HStore(327): Store=a54460b3855674c4d88c6c0a82684355/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T08:38:06,214 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,216 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,216 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,217 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,217 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,221 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,225 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T08:38:06,226 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened a54460b3855674c4d88c6c0a82684355; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72229379, jitterRate=0.07630161941051483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T08:38:06,226 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:06,228 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for a54460b3855674c4d88c6c0a82684355: Running coprocessor pre-open hook at 1732783086208Writing region info on filesystem at 1732783086208Initializing all the Stores at 1732783086210 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732783086210Cleaning up temporary data from old regions at 1732783086217 (+7 ms)Running coprocessor post-open hooks at 1732783086226 (+9 ms)Region opened successfully at 1732783086228 (+2 ms) 2024-11-28T08:38:06,229 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., pid=33, masterSystemTime=1732783086193 2024-11-28T08:38:06,232 DEBUG [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:06,232 INFO [RS_OPEN_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:06,233 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=a54460b3855674c4d88c6c0a82684355, regionState=OPEN, openSeqNum=2, regionLocation=81db972b9a1d,32805,1732782990425 2024-11-28T08:38:06,236 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure a54460b3855674c4d88c6c0a82684355, server=81db972b9a1d,32805,1732782990425 because future has completed 2024-11-28T08:38:06,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T08:38:06,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure a54460b3855674c4d88c6c0a82684355, server=81db972b9a1d,32805,1732782990425 in 199 msec 2024-11-28T08:38:06,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-28T08:38:06,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a54460b3855674c4d88c6c0a82684355, ASSIGN in 360 msec 2024-11-28T08:38:06,244 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T08:38:06,244 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732783086244"}]},"ts":"1732783086244"} 2024-11-28T08:38:06,246 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-28T08:38:06,247 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T08:38:06,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2220 sec 2024-11-28T08:38:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T08:38:08,061 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-28T08:38:08,062 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-28T08:38:08,062 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:38:08,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-28T08:38:08,073 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T08:38:08,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-28T08:38:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:38:08,085 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T08:38:08,085 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-28T08:38:08,088 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T08:38:08,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T08:38:09,095 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=81db972b9a1d:32805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-28T08:38:09,095 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-28T08:38:09,095 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 from cache 2024-11-28T08:38:09,120 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-28T08:38:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-28T08:38:10,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-28T08:38:10,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T08:38:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2] 2024-11-28T08:38:10,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T08:38:11,095 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=81db972b9a1d:32805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=976ms 2024-11-28T08:38:11,095 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=976ms 2024-11-28T08:38:11,096 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., hostname=81db972b9a1d,32805,1732782990425, seqNum=2 from cache 2024-11-28T08:38:11,112 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=95 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=6584 (was 6590) 2024-11-28T08:38:11,113 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T08:38:11,113 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T08:38:11,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:11,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:11,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T08:38:11,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T08:38:11,119 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T08:38:11,120 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T08:38:11,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:11,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:11,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T08:38:11,120 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T08:38:11,120 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2119115424, stopped=false 2024-11-28T08:38:11,121 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=81db972b9a1d,36019,1732782989669 2024-11-28T08:38:11,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T08:38:11,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T08:38:11,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:38:11,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:38:11,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T08:38:11,195 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T08:38:11,195 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T08:38:11,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:11,196 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T08:38:11,196 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '81db972b9a1d,32805,1732782990425' ***** 2024-11-28T08:38:11,197 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T08:38:11,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T08:38:11,197 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T08:38:11,198 INFO [RS:0;81db972b9a1d:32805 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T08:38:11,198 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T08:38:11,198 INFO [RS:0;81db972b9a1d:32805 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T08:38:11,198 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for 38932b92ba0f789ca2e99146b63e23c3 2024-11-28T08:38:11,199 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:38:11,199 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for a54460b3855674c4d88c6c0a82684355 2024-11-28T08:38:11,199 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for a1413cdeef46e22d3ec726a59a54296b 2024-11-28T08:38:11,199 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for 46db51c6773fee2566ff7a311e132c68 2024-11-28T08:38:11,199 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for a1d5c220899b33f9a0e277860d1eff68 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for 5a10d1d28856433fd41135169cd86b59 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:11,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38932b92ba0f789ca2e99146b63e23c3, disabling compactions & flushes 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for d0ff3c4719c548dd9b9b6a0fdddf645c 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for d4af0876fb90af949d407490d72c4923 2024-11-28T08:38:11,200 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:38:11,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(959): stopping server 81db972b9a1d,32805,1732782990425 2024-11-28T08:38:11,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. after waiting 0 ms 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T08:38:11,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;81db972b9a1d:32805. 2024-11-28T08:38:11,200 DEBUG [RS:0;81db972b9a1d:32805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T08:38:11,200 DEBUG [RS:0;81db972b9a1d:32805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:11,200 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T08:38:11,201 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T08:38:11,201 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T08:38:11,201 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T08:38:11,201 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-28T08:38:11,201 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T08:38:11,201 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T08:38:11,201 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T08:38:11,201 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T08:38:11,201 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T08:38:11,201 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 38932b92ba0f789ca2e99146b63e23c3, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d, f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:38:11,204 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 38932b92ba0f789ca2e99146b63e23c3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:11,204 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-28T08:38:11,255 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/.tmp/testFamily/2e87cd2859374cd782be61d81fa56ea5 is 61, key is row-0/testFamily:testQualifier/1732783039372/Put/seqid=0 2024-11-28T08:38:11,261 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/info/0ab77b958fbf49cb8d8e0eb32669eda0 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b./info:regioninfo/1732783025241/Put/seqid=0 2024-11-28T08:38:11,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741855_1031 (size=5049) 2024-11-28T08:38:11,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741856_1032 (size=20564) 2024-11-28T08:38:11,402 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 38932b92ba0f789ca2e99146b63e23c3, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d, f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:38:11,603 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 38932b92ba0f789ca2e99146b63e23c3, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d, f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:38:11,615 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-28T08:38:11,667 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/.tmp/testFamily/2e87cd2859374cd782be61d81fa56ea5 2024-11-28T08:38:11,672 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/info/0ab77b958fbf49cb8d8e0eb32669eda0 2024-11-28T08:38:11,708 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:38:11,721 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/.tmp/testFamily/2e87cd2859374cd782be61d81fa56ea5 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/testFamily/2e87cd2859374cd782be61d81fa56ea5 2024-11-28T08:38:11,730 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/testFamily/2e87cd2859374cd782be61d81fa56ea5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:11,739 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/ns/1b523e6ecc124deba3738d88245e07a0 is 43, key is default/ns:d/1732783023749/Put/seqid=0 2024-11-28T08:38:11,740 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 38932b92ba0f789ca2e99146b63e23c3 in 534ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:11,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741857_1033 (size=5153) 2024-11-28T08:38:11,748 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNextAsync/38932b92ba0f789ca2e99146b63e23c3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:11,750 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:38:11,750 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38932b92ba0f789ca2e99146b63e23c3: Waiting for close lock at 1732783091199Running coprocessor pre-close hooks at 1732783091200 (+1 ms)Disabling compacts and flushes for region at 1732783091200Disabling writes for close at 1732783091200Obtaining lock to block concurrent updates at 1732783091204 (+4 ms)Preparing flush snapshotting stores in 38932b92ba0f789ca2e99146b63e23c3 at 1732783091204Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783091211 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. at 1732783091212 (+1 ms)Flushing 38932b92ba0f789ca2e99146b63e23c3/testFamily: creating writer at 1732783091214 (+2 ms)Flushing 38932b92ba0f789ca2e99146b63e23c3/testFamily: appending metadata at 1732783091250 (+36 ms)Flushing 38932b92ba0f789ca2e99146b63e23c3/testFamily: closing flushed file at 1732783091252 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de1327b: reopening flushed file at 1732783091719 (+467 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 38932b92ba0f789ca2e99146b63e23c3 in 534ms, sequenceid=8, compaction requested=false at 1732783091740 (+21 ms)Writing region close event to WAL at 1732783091742 (+2 ms)Running coprocessor post-close hooks at 1732783091749 (+7 ms)Closed at 1732783091750 (+1 ms) 2024-11-28T08:38:11,751 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732783036294.38932b92ba0f789ca2e99146b63e23c3. 2024-11-28T08:38:11,751 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f6d20656c001a8c21b64a85595a60dcd, disabling compactions & flushes 2024-11-28T08:38:11,751 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:38:11,751 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:38:11,751 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. after waiting 0 ms 2024-11-28T08:38:11,751 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:38:11,751 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f6d20656c001a8c21b64a85595a60dcd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:11,768 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/.tmp/testFamily/0aab9fc92759485a8163ef41911a5aaa is 61, key is row-0/testFamily:testQualifier/1732783048480/Put/seqid=0 2024-11-28T08:38:11,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741858_1034 (size=5049) 2024-11-28T08:38:11,803 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d, f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:38:12,003 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d, f6d20656c001a8c21b64a85595a60dcd 2024-11-28T08:38:12,005 INFO [regionserver/81db972b9a1d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T08:38:12,005 INFO [regionserver/81db972b9a1d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T08:38:12,146 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/ns/1b523e6ecc124deba3738d88245e07a0 2024-11-28T08:38:12,174 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/.tmp/testFamily/0aab9fc92759485a8163ef41911a5aaa 2024-11-28T08:38:12,183 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/table/c6dc32ac44c34c83a7e5a31d3c8eb2ae is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732783025258/Put/seqid=0 2024-11-28T08:38:12,184 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/.tmp/testFamily/0aab9fc92759485a8163ef41911a5aaa as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/testFamily/0aab9fc92759485a8163ef41911a5aaa 2024-11-28T08:38:12,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741859_1035 (size=6699) 2024-11-28T08:38:12,193 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/testFamily/0aab9fc92759485a8163ef41911a5aaa, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:12,194 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f6d20656c001a8c21b64a85595a60dcd in 443ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:12,199 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextException/f6d20656c001a8c21b64a85595a60dcd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:12,200 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:38:12,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f6d20656c001a8c21b64a85595a60dcd: Waiting for close lock at 1732783091751Running coprocessor pre-close hooks at 1732783091751Disabling compacts and flushes for region at 1732783091751Disabling writes for close at 1732783091751Obtaining lock to block concurrent updates at 1732783091751Preparing flush snapshotting stores in f6d20656c001a8c21b64a85595a60dcd at 1732783091751Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783091751Flushing stores of testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. at 1732783091752 (+1 ms)Flushing f6d20656c001a8c21b64a85595a60dcd/testFamily: creating writer at 1732783091752Flushing f6d20656c001a8c21b64a85595a60dcd/testFamily: appending metadata at 1732783091768 (+16 ms)Flushing f6d20656c001a8c21b64a85595a60dcd/testFamily: closing flushed file at 1732783091768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f70737e: reopening flushed file at 1732783092182 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f6d20656c001a8c21b64a85595a60dcd in 443ms, sequenceid=8, compaction requested=false at 1732783092194 (+12 ms)Writing region close event to WAL at 1732783092195 (+1 ms)Running coprocessor post-close hooks at 1732783092200 (+5 ms)Closed at 1732783092200 2024-11-28T08:38:12,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732783045416.f6d20656c001a8c21b64a85595a60dcd. 2024-11-28T08:38:12,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a54460b3855674c4d88c6c0a82684355, disabling compactions & flushes 2024-11-28T08:38:12,200 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:12,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:12,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. after waiting 0 ms 2024-11-28T08:38:12,200 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:12,200 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a54460b3855674c4d88c6c0a82684355 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:12,204 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-28T08:38:12,204 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1325): Online Regions={a54460b3855674c4d88c6c0a82684355=testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., a1413cdeef46e22d3ec726a59a54296b=testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b., 46db51c6773fee2566ff7a311e132c68=hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68., a1d5c220899b33f9a0e277860d1eff68=testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68., 5a10d1d28856433fd41135169cd86b59=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., 1588230740=hbase:meta,,1.1588230740, e24f2da55cdb9e86fc742191c3a6234d=testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d., d0ff3c4719c548dd9b9b6a0fdddf645c=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., d4af0876fb90af949d407490d72c4923=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.} 2024-11-28T08:38:12,204 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:12,217 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/.tmp/testFamily/ce6d7f0a94b84d64923d1d9eff238ca6 is 61, key is row-0/testFamily:testQualifier/1732783088076/Put/seqid=0 2024-11-28T08:38:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741860_1036 (size=5049) 2024-11-28T08:38:12,404 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:12,591 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/table/c6dc32ac44c34c83a7e5a31d3c8eb2ae 2024-11-28T08:38:12,604 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, a54460b3855674c4d88c6c0a82684355, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:12,608 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/info/0ab77b958fbf49cb8d8e0eb32669eda0 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/info/0ab77b958fbf49cb8d8e0eb32669eda0 2024-11-28T08:38:12,618 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/info/0ab77b958fbf49cb8d8e0eb32669eda0, entries=100, sequenceid=56, filesize=20.1 K 2024-11-28T08:38:12,620 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/ns/1b523e6ecc124deba3738d88245e07a0 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/ns/1b523e6ecc124deba3738d88245e07a0 2024-11-28T08:38:12,623 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/.tmp/testFamily/ce6d7f0a94b84d64923d1d9eff238ca6 2024-11-28T08:38:12,629 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/ns/1b523e6ecc124deba3738d88245e07a0, entries=2, sequenceid=56, filesize=5.0 K 2024-11-28T08:38:12,630 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/.tmp/testFamily/ce6d7f0a94b84d64923d1d9eff238ca6 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/testFamily/ce6d7f0a94b84d64923d1d9eff238ca6 2024-11-28T08:38:12,631 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/.tmp/table/c6dc32ac44c34c83a7e5a31d3c8eb2ae as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/table/c6dc32ac44c34c83a7e5a31d3c8eb2ae 2024-11-28T08:38:12,638 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/testFamily/ce6d7f0a94b84d64923d1d9eff238ca6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:12,638 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/table/c6dc32ac44c34c83a7e5a31d3c8eb2ae, entries=20, sequenceid=56, filesize=6.5 K 2024-11-28T08:38:12,639 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a54460b3855674c4d88c6c0a82684355 in 439ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:12,639 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1438ms, sequenceid=56, compaction requested=false 2024-11-28T08:38:12,644 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScannerAsync/a54460b3855674c4d88c6c0a82684355/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:12,644 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-28T08:38:12,645 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T08:38:12,645 INFO [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T08:38:12,645 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:12,645 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732783091201Running coprocessor pre-close hooks at 1732783091201Disabling compacts and flushes for region at 1732783091201Disabling writes for close at 1732783091201Obtaining lock to block concurrent updates at 1732783091204 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732783091204Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732783091211 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732783091212 (+1 ms)Flushing 1588230740/info: creating writer at 1732783091214 (+2 ms)Flushing 1588230740/info: appending metadata at 1732783091255 (+41 ms)Flushing 1588230740/info: closing flushed file at 1732783091256 (+1 ms)Flushing 1588230740/ns: creating writer at 1732783091719 (+463 ms)Flushing 1588230740/ns: appending metadata at 1732783091738 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732783091738Flushing 1588230740/table: creating writer at 1732783092160 (+422 ms)Flushing 1588230740/table: appending metadata at 1732783092182 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732783092182Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d88c77b: reopening flushed file at 1732783092607 (+425 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c9b5a2: reopening flushed file at 1732783092618 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b912a16: reopening flushed file at 1732783092630 (+12 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1438ms, sequenceid=56, compaction requested=false at 1732783092639 (+9 ms)Writing region close event to WAL at 1732783092641 (+2 ms)Running coprocessor post-close hooks at 1732783092645 (+4 ms)Closed at 1732783092645 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a54460b3855674c4d88c6c0a82684355: Waiting for close lock at 1732783092200Running coprocessor pre-close hooks at 1732783092200Disabling compacts and flushes for region at 1732783092200Disabling writes for close at 1732783092200Obtaining lock to block concurrent updates at 1732783092201 (+1 ms)Preparing flush snapshotting stores in a54460b3855674c4d88c6c0a82684355 at 1732783092201Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783092201Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. at 1732783092201Flushing a54460b3855674c4d88c6c0a82684355/testFamily: creating writer at 1732783092201Flushing a54460b3855674c4d88c6c0a82684355/testFamily: appending metadata at 1732783092216 (+15 ms)Flushing a54460b3855674c4d88c6c0a82684355/testFamily: closing flushed file at 1732783092216Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44655097: reopening flushed file at 1732783092629 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a54460b3855674c4d88c6c0a82684355 in 439ms, sequenceid=8, compaction requested=false at 1732783092639 (+10 ms)Writing region close event to WAL at 1732783092640 (+1 ms)Running coprocessor post-close hooks at 1732783092645 (+5 ms)Closed at 1732783092645 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_META-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732783085025.a54460b3855674c4d88c6c0a82684355. 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a1413cdeef46e22d3ec726a59a54296b, disabling compactions & flushes 2024-11-28T08:38:12,646 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. after waiting 0 ms 2024-11-28T08:38:12,646 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:38:12,646 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a1413cdeef46e22d3ec726a59a54296b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:12,662 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/.tmp/testFamily/58c8399515654e02b5033a89131b4fa8 is 61, key is row-0/testFamily:testQualifier/1732783027095/Put/seqid=0 2024-11-28T08:38:12,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741861_1037 (size=5049) 2024-11-28T08:38:12,805 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:13,005 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1413cdeef46e22d3ec726a59a54296b, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:13,069 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/.tmp/testFamily/58c8399515654e02b5033a89131b4fa8 2024-11-28T08:38:13,084 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/.tmp/testFamily/58c8399515654e02b5033a89131b4fa8 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/testFamily/58c8399515654e02b5033a89131b4fa8 2024-11-28T08:38:13,092 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/testFamily/58c8399515654e02b5033a89131b4fa8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:13,093 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a1413cdeef46e22d3ec726a59a54296b in 447ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:13,097 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a1413cdeef46e22d3ec726a59a54296b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:13,098 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:38:13,098 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a1413cdeef46e22d3ec726a59a54296b: Waiting for close lock at 1732783092646Running coprocessor pre-close hooks at 1732783092646Disabling compacts and flushes for region at 1732783092646Disabling writes for close at 1732783092646Obtaining lock to block concurrent updates at 1732783092646Preparing flush snapshotting stores in a1413cdeef46e22d3ec726a59a54296b at 1732783092646Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783092646Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. at 1732783092647 (+1 ms)Flushing a1413cdeef46e22d3ec726a59a54296b/testFamily: creating writer at 1732783092647Flushing a1413cdeef46e22d3ec726a59a54296b/testFamily: appending metadata at 1732783092662 (+15 ms)Flushing a1413cdeef46e22d3ec726a59a54296b/testFamily: closing flushed file at 1732783092662Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772f4a5b: reopening flushed file at 1732783093083 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a1413cdeef46e22d3ec726a59a54296b in 447ms, sequenceid=8, compaction requested=false at 1732783093093 (+10 ms)Writing region close event to WAL at 1732783093093Running coprocessor post-close hooks at 1732783093098 (+5 ms)Closed at 1732783093098 2024-11-28T08:38:13,098 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732783023966.a1413cdeef46e22d3ec726a59a54296b. 2024-11-28T08:38:13,098 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46db51c6773fee2566ff7a311e132c68, disabling compactions & flushes 2024-11-28T08:38:13,098 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:38:13,098 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:38:13,098 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. after waiting 0 ms 2024-11-28T08:38:13,098 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:38:13,098 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 46db51c6773fee2566ff7a311e132c68 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:13,115 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/.tmp/testFamily/d2a451dcd6e0400fa2b8c72d5ee67e45 is 61, key is row-0/testFamily:testQualifier/1732783078977/Put/seqid=0 2024-11-28T08:38:13,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741862_1038 (size=5049) 2024-11-28T08:38:13,206 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-28T08:38:13,206 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1325): Online Regions={46db51c6773fee2566ff7a311e132c68=hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68., a1d5c220899b33f9a0e277860d1eff68=testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68., 5a10d1d28856433fd41135169cd86b59=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., e24f2da55cdb9e86fc742191c3a6234d=testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d., d0ff3c4719c548dd9b9b6a0fdddf645c=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., d4af0876fb90af949d407490d72c4923=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.} 2024-11-28T08:38:13,206 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:13,406 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 46db51c6773fee2566ff7a311e132c68, 5a10d1d28856433fd41135169cd86b59, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:13,522 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/.tmp/testFamily/d2a451dcd6e0400fa2b8c72d5ee67e45 2024-11-28T08:38:13,537 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/.tmp/testFamily/d2a451dcd6e0400fa2b8c72d5ee67e45 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/testFamily/d2a451dcd6e0400fa2b8c72d5ee67e45 2024-11-28T08:38:13,546 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/testFamily/d2a451dcd6e0400fa2b8c72d5ee67e45, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:13,548 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46db51c6773fee2566ff7a311e132c68 in 450ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:13,552 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnNext/46db51c6773fee2566ff7a311e132c68/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:13,553 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:38:13,553 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46db51c6773fee2566ff7a311e132c68: Waiting for close lock at 1732783093098Running coprocessor pre-close hooks at 1732783093098Disabling compacts and flushes for region at 1732783093098Disabling writes for close at 1732783093098Obtaining lock to block concurrent updates at 1732783093098Preparing flush snapshotting stores in 46db51c6773fee2566ff7a311e132c68 at 1732783093098Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783093099 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. at 1732783093099Flushing 46db51c6773fee2566ff7a311e132c68/testFamily: creating writer at 1732783093099Flushing 46db51c6773fee2566ff7a311e132c68/testFamily: appending metadata at 1732783093115 (+16 ms)Flushing 46db51c6773fee2566ff7a311e132c68/testFamily: closing flushed file at 1732783093115Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eb6e04c: reopening flushed file at 1732783093536 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46db51c6773fee2566ff7a311e132c68 in 450ms, sequenceid=8, compaction requested=false at 1732783093548 (+12 ms)Writing region close event to WAL at 1732783093548Running coprocessor post-close hooks at 1732783093553 (+5 ms)Closed at 1732783093553 2024-11-28T08:38:13,553 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732783075933.46db51c6773fee2566ff7a311e132c68. 2024-11-28T08:38:13,554 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a1d5c220899b33f9a0e277860d1eff68, disabling compactions & flushes 2024-11-28T08:38:13,554 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:38:13,554 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:38:13,554 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. after waiting 0 ms 2024-11-28T08:38:13,554 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:38:13,554 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a1d5c220899b33f9a0e277860d1eff68 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:13,569 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/.tmp/testFamily/a3ee53dd58ea48b29982699e83d03549 is 61, key is row-0/testFamily:testQualifier/1732783057683/Put/seqid=0 2024-11-28T08:38:13,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741863_1039 (size=5049) 2024-11-28T08:38:13,607 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 5a10d1d28856433fd41135169cd86b59, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:13,807 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 5a10d1d28856433fd41135169cd86b59, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:13,977 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/.tmp/testFamily/a3ee53dd58ea48b29982699e83d03549 2024-11-28T08:38:14,008 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 5a10d1d28856433fd41135169cd86b59, a1d5c220899b33f9a0e277860d1eff68, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:14,032 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/.tmp/testFamily/a3ee53dd58ea48b29982699e83d03549 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/testFamily/a3ee53dd58ea48b29982699e83d03549 2024-11-28T08:38:14,043 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/testFamily/a3ee53dd58ea48b29982699e83d03549, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:14,044 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a1d5c220899b33f9a0e277860d1eff68 in 490ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:14,048 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNextAsync/a1d5c220899b33f9a0e277860d1eff68/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:14,049 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:38:14,049 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a1d5c220899b33f9a0e277860d1eff68: Waiting for close lock at 1732783093553Running coprocessor pre-close hooks at 1732783093553Disabling compacts and flushes for region at 1732783093553Disabling writes for close at 1732783093554 (+1 ms)Obtaining lock to block concurrent updates at 1732783093554Preparing flush snapshotting stores in a1d5c220899b33f9a0e277860d1eff68 at 1732783093554Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783093554Flushing stores of testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. at 1732783093555 (+1 ms)Flushing a1d5c220899b33f9a0e277860d1eff68/testFamily: creating writer at 1732783093555Flushing a1d5c220899b33f9a0e277860d1eff68/testFamily: appending metadata at 1732783093569 (+14 ms)Flushing a1d5c220899b33f9a0e277860d1eff68/testFamily: closing flushed file at 1732783093569Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a5ba55: reopening flushed file at 1732783094030 (+461 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a1d5c220899b33f9a0e277860d1eff68 in 490ms, sequenceid=8, compaction requested=false at 1732783094044 (+14 ms)Writing region close event to WAL at 1732783094045 (+1 ms)Running coprocessor post-close hooks at 1732783094049 (+4 ms)Closed at 1732783094049 2024-11-28T08:38:14,049 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732783054630.a1d5c220899b33f9a0e277860d1eff68. 2024-11-28T08:38:14,049 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5a10d1d28856433fd41135169cd86b59, disabling compactions & flushes 2024-11-28T08:38:14,049 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:38:14,049 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:38:14,050 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. after waiting 0 ms 2024-11-28T08:38:14,050 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:38:14,050 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5a10d1d28856433fd41135169cd86b59 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:14,068 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/.tmp/testFamily/832762a20e104c2c9199a2ad5743a770 is 61, key is row-0/testFamily:testQualifier/1732783030249/Put/seqid=0 2024-11-28T08:38:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741864_1040 (size=5049) 2024-11-28T08:38:14,208 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-28T08:38:14,208 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1325): Online Regions={5a10d1d28856433fd41135169cd86b59=hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., e24f2da55cdb9e86fc742191c3a6234d=testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d., d0ff3c4719c548dd9b9b6a0fdddf645c=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., d4af0876fb90af949d407490d72c4923=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.} 2024-11-28T08:38:14,209 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 5a10d1d28856433fd41135169cd86b59, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:14,409 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on 5a10d1d28856433fd41135169cd86b59, d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:14,476 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/.tmp/testFamily/832762a20e104c2c9199a2ad5743a770 2024-11-28T08:38:14,491 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/.tmp/testFamily/832762a20e104c2c9199a2ad5743a770 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/testFamily/832762a20e104c2c9199a2ad5743a770 2024-11-28T08:38:14,497 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/testFamily/832762a20e104c2c9199a2ad5743a770, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:14,498 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5a10d1d28856433fd41135169cd86b59 in 448ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:14,503 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScanner/5a10d1d28856433fd41135169cd86b59/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:14,504 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:38:14,504 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5a10d1d28856433fd41135169cd86b59: Waiting for close lock at 1732783094049Running coprocessor pre-close hooks at 1732783094049Disabling compacts and flushes for region at 1732783094049Disabling writes for close at 1732783094050 (+1 ms)Obtaining lock to block concurrent updates at 1732783094050Preparing flush snapshotting stores in 5a10d1d28856433fd41135169cd86b59 at 1732783094050Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783094050Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. at 1732783094051 (+1 ms)Flushing 5a10d1d28856433fd41135169cd86b59/testFamily: creating writer at 1732783094051Flushing 5a10d1d28856433fd41135169cd86b59/testFamily: appending metadata at 1732783094068 (+17 ms)Flushing 5a10d1d28856433fd41135169cd86b59/testFamily: closing flushed file at 1732783094068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e74c7a4: reopening flushed file at 1732783094490 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5a10d1d28856433fd41135169cd86b59 in 448ms, sequenceid=8, compaction requested=false at 1732783094498 (+8 ms)Writing region close event to WAL at 1732783094499 (+1 ms)Running coprocessor post-close hooks at 1732783094504 (+5 ms)Closed at 1732783094504 2024-11-28T08:38:14,505 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732783027187.5a10d1d28856433fd41135169cd86b59. 2024-11-28T08:38:14,505 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e24f2da55cdb9e86fc742191c3a6234d, disabling compactions & flushes 2024-11-28T08:38:14,505 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:38:14,505 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:38:14,505 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. after waiting 0 ms 2024-11-28T08:38:14,505 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:38:14,505 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e24f2da55cdb9e86fc742191c3a6234d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:14,525 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/.tmp/testFamily/3a3fa9381c7941288ec27becda612066 is 61, key is row-0/testFamily:testQualifier/1732783063780/Put/seqid=0 2024-11-28T08:38:14,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741865_1041 (size=5049) 2024-11-28T08:38:14,609 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:14,810 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923, e24f2da55cdb9e86fc742191c3a6234d 2024-11-28T08:38:14,933 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/.tmp/testFamily/3a3fa9381c7941288ec27becda612066 2024-11-28T08:38:14,946 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/.tmp/testFamily/3a3fa9381c7941288ec27becda612066 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/testFamily/3a3fa9381c7941288ec27becda612066 2024-11-28T08:38:14,953 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/testFamily/3a3fa9381c7941288ec27becda612066, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:14,954 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e24f2da55cdb9e86fc742191c3a6234d in 449ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:14,959 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnNext/e24f2da55cdb9e86fc742191c3a6234d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:14,959 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:38:14,960 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e24f2da55cdb9e86fc742191c3a6234d: Waiting for close lock at 1732783094505Running coprocessor pre-close hooks at 1732783094505Disabling compacts and flushes for region at 1732783094505Disabling writes for close at 1732783094505Obtaining lock to block concurrent updates at 1732783094505Preparing flush snapshotting stores in e24f2da55cdb9e86fc742191c3a6234d at 1732783094505Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783094505Flushing stores of testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. at 1732783094506 (+1 ms)Flushing e24f2da55cdb9e86fc742191c3a6234d/testFamily: creating writer at 1732783094506Flushing e24f2da55cdb9e86fc742191c3a6234d/testFamily: appending metadata at 1732783094525 (+19 ms)Flushing e24f2da55cdb9e86fc742191c3a6234d/testFamily: closing flushed file at 1732783094525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d0444db: reopening flushed file at 1732783094945 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e24f2da55cdb9e86fc742191c3a6234d in 449ms, sequenceid=8, compaction requested=false at 1732783094954 (+9 ms)Writing region close event to WAL at 1732783094955 (+1 ms)Running coprocessor post-close hooks at 1732783094959 (+4 ms)Closed at 1732783094959 2024-11-28T08:38:14,960 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732783060728.e24f2da55cdb9e86fc742191c3a6234d. 2024-11-28T08:38:14,960 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d0ff3c4719c548dd9b9b6a0fdddf645c, disabling compactions & flushes 2024-11-28T08:38:14,960 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:38:14,960 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:38:14,960 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. after waiting 0 ms 2024-11-28T08:38:14,960 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:38:14,960 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d0ff3c4719c548dd9b9b6a0fdddf645c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:14,976 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/.tmp/testFamily/c1e69a432a294cd8bbb4e4379e2b7250 is 61, key is row-0/testFamily:testQualifier/1732783069888/Put/seqid=0 2024-11-28T08:38:14,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741866_1042 (size=5049) 2024-11-28T08:38:15,010 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923 2024-11-28T08:38:15,211 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T08:38:15,211 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1325): Online Regions={d0ff3c4719c548dd9b9b6a0fdddf645c=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., d4af0876fb90af949d407490d72c4923=testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923.} 2024-11-28T08:38:15,211 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d0ff3c4719c548dd9b9b6a0fdddf645c, d4af0876fb90af949d407490d72c4923 2024-11-28T08:38:15,384 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/.tmp/testFamily/c1e69a432a294cd8bbb4e4379e2b7250 2024-11-28T08:38:15,395 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/.tmp/testFamily/c1e69a432a294cd8bbb4e4379e2b7250 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/testFamily/c1e69a432a294cd8bbb4e4379e2b7250 2024-11-28T08:38:15,402 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/testFamily/c1e69a432a294cd8bbb4e4379e2b7250, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:15,403 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d0ff3c4719c548dd9b9b6a0fdddf645c in 443ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:15,408 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d0ff3c4719c548dd9b9b6a0fdddf645c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:15,409 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:38:15,409 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d0ff3c4719c548dd9b9b6a0fdddf645c: Waiting for close lock at 1732783094960Running coprocessor pre-close hooks at 1732783094960Disabling compacts and flushes for region at 1732783094960Disabling writes for close at 1732783094960Obtaining lock to block concurrent updates at 1732783094960Preparing flush snapshotting stores in d0ff3c4719c548dd9b9b6a0fdddf645c at 1732783094960Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783094960Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. at 1732783094961 (+1 ms)Flushing d0ff3c4719c548dd9b9b6a0fdddf645c/testFamily: creating writer at 1732783094961Flushing d0ff3c4719c548dd9b9b6a0fdddf645c/testFamily: appending metadata at 1732783094976 (+15 ms)Flushing d0ff3c4719c548dd9b9b6a0fdddf645c/testFamily: closing flushed file at 1732783094976Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a76a398: reopening flushed file at 1732783095394 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d0ff3c4719c548dd9b9b6a0fdddf645c in 443ms, sequenceid=8, compaction requested=false at 1732783095403 (+9 ms)Writing region close event to WAL at 1732783095404 (+1 ms)Running coprocessor post-close hooks at 1732783095408 (+4 ms)Closed at 1732783095409 (+1 ms) 2024-11-28T08:38:15,409 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732783066835.d0ff3c4719c548dd9b9b6a0fdddf645c. 2024-11-28T08:38:15,409 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d4af0876fb90af949d407490d72c4923, disabling compactions & flushes 2024-11-28T08:38:15,409 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:38:15,409 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:38:15,409 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. after waiting 0 ms 2024-11-28T08:38:15,409 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:38:15,409 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d4af0876fb90af949d407490d72c4923 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T08:38:15,411 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d4af0876fb90af949d407490d72c4923 2024-11-28T08:38:15,425 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/.tmp/testFamily/771219b5dd044da882d5ae4065bb03a4 is 61, key is row-0/testFamily:testQualifier/1732783051580/Put/seqid=0 2024-11-28T08:38:15,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741867_1043 (size=5049) 2024-11-28T08:38:15,611 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d4af0876fb90af949d407490d72c4923 2024-11-28T08:38:15,812 DEBUG [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1351): Waiting on d4af0876fb90af949d407490d72c4923 2024-11-28T08:38:15,832 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/.tmp/testFamily/771219b5dd044da882d5ae4065bb03a4 2024-11-28T08:38:15,847 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/.tmp/testFamily/771219b5dd044da882d5ae4065bb03a4 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/testFamily/771219b5dd044da882d5ae4065bb03a4 2024-11-28T08:38:15,854 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/testFamily/771219b5dd044da882d5ae4065bb03a4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T08:38:15,855 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d4af0876fb90af949d407490d72c4923 in 446ms, sequenceid=8, compaction requested=false 2024-11-28T08:38:15,859 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/data/default/testNormalScanTimeoutOnOpenScanner/d4af0876fb90af949d407490d72c4923/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T08:38:15,860 INFO [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:38:15,860 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d4af0876fb90af949d407490d72c4923: Waiting for close lock at 1732783095409Running coprocessor pre-close hooks at 1732783095409Disabling compacts and flushes for region at 1732783095409Disabling writes for close at 1732783095409Obtaining lock to block concurrent updates at 1732783095409Preparing flush snapshotting stores in d4af0876fb90af949d407490d72c4923 at 1732783095409Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732783095410 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. at 1732783095410Flushing d4af0876fb90af949d407490d72c4923/testFamily: creating writer at 1732783095410Flushing d4af0876fb90af949d407490d72c4923/testFamily: appending metadata at 1732783095425 (+15 ms)Flushing d4af0876fb90af949d407490d72c4923/testFamily: closing flushed file at 1732783095425Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7de78662: reopening flushed file at 1732783095845 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d4af0876fb90af949d407490d72c4923 in 446ms, sequenceid=8, compaction requested=false at 1732783095855 (+10 ms)Writing region close event to WAL at 1732783095856 (+1 ms)Running coprocessor post-close hooks at 1732783095860 (+4 ms)Closed at 1732783095860 2024-11-28T08:38:15,860 DEBUG [RS_CLOSE_REGION-regionserver/81db972b9a1d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732783048535.d4af0876fb90af949d407490d72c4923. 2024-11-28T08:38:16,013 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(976): stopping server 81db972b9a1d,32805,1732782990425; all regions closed. 2024-11-28T08:38:16,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741834_1010 (size=19479) 2024-11-28T08:38:16,033 DEBUG [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/oldWALs 2024-11-28T08:38:16,033 INFO [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 81db972b9a1d%2C32805%2C1732782990425.meta:.meta(num 1732783023481) 2024-11-28T08:38:16,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741833_1009 (size=20031) 2024-11-28T08:38:16,036 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/WALs/81db972b9a1d,32805,1732782990425/81db972b9a1d%2C32805%2C1732782990425.1732783023064 not finished, retry = 0 2024-11-28T08:38:16,145 DEBUG [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/oldWALs 2024-11-28T08:38:16,145 INFO [RS:0;81db972b9a1d:32805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 81db972b9a1d%2C32805%2C1732782990425:(num 1732783023064) 2024-11-28T08:38:16,145 DEBUG [RS:0;81db972b9a1d:32805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T08:38:16,145 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T08:38:16,145 INFO [RS:0;81db972b9a1d:32805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T08:38:16,146 INFO [RS:0;81db972b9a1d:32805 {}] hbase.ChoreService(370): Chore service for: regionserver/81db972b9a1d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T08:38:16,146 INFO [RS:0;81db972b9a1d:32805 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T08:38:16,146 INFO [regionserver/81db972b9a1d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T08:38:16,148 INFO [RS:0;81db972b9a1d:32805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32805 2024-11-28T08:38:16,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/81db972b9a1d,32805,1732782990425 2024-11-28T08:38:16,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T08:38:16,213 INFO [RS:0;81db972b9a1d:32805 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T08:38:16,215 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [81db972b9a1d,32805,1732782990425] 2024-11-28T08:38:16,231 INFO [regionserver/81db972b9a1d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T08:38:16,234 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/81db972b9a1d,32805,1732782990425 already deleted, retry=false 2024-11-28T08:38:16,235 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 81db972b9a1d,32805,1732782990425 expired; onlineServers=0 2024-11-28T08:38:16,235 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '81db972b9a1d,36019,1732782989669' ***** 2024-11-28T08:38:16,235 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T08:38:16,235 INFO [M:0;81db972b9a1d:36019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T08:38:16,236 INFO [M:0;81db972b9a1d:36019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T08:38:16,236 DEBUG [M:0;81db972b9a1d:36019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T08:38:16,236 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T08:38:16,237 DEBUG [M:0;81db972b9a1d:36019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T08:38:16,237 DEBUG [master/81db972b9a1d:0:becomeActiveMaster-HFileCleaner.large.0-1732782992867 {}] cleaner.HFileCleaner(306): Exit Thread[master/81db972b9a1d:0:becomeActiveMaster-HFileCleaner.large.0-1732782992867,5,FailOnTimeoutGroup] 2024-11-28T08:38:16,237 DEBUG [master/81db972b9a1d:0:becomeActiveMaster-HFileCleaner.small.0-1732782992869 {}] cleaner.HFileCleaner(306): Exit Thread[master/81db972b9a1d:0:becomeActiveMaster-HFileCleaner.small.0-1732782992869,5,FailOnTimeoutGroup] 2024-11-28T08:38:16,237 INFO [M:0;81db972b9a1d:36019 {}] hbase.ChoreService(370): Chore service for: master/81db972b9a1d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T08:38:16,238 INFO [M:0;81db972b9a1d:36019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T08:38:16,238 DEBUG [M:0;81db972b9a1d:36019 {}] master.HMaster(1795): Stopping service threads 2024-11-28T08:38:16,238 INFO [M:0;81db972b9a1d:36019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T08:38:16,238 INFO [M:0;81db972b9a1d:36019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T08:38:16,240 INFO [M:0;81db972b9a1d:36019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T08:38:16,240 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T08:38:16,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T08:38:16,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T08:38:16,245 DEBUG [M:0;81db972b9a1d:36019 {}] zookeeper.ZKUtil(347): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T08:38:16,245 WARN [M:0;81db972b9a1d:36019 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T08:38:16,248 INFO [M:0;81db972b9a1d:36019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/.lastflushedseqids 2024-11-28T08:38:16,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741868_1044 (size=649) 2024-11-28T08:38:16,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T08:38:16,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32805-0x101809619d40001, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T08:38:16,326 INFO [RS:0;81db972b9a1d:32805 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T08:38:16,326 INFO [RS:0;81db972b9a1d:32805 {}] regionserver.HRegionServer(1031): Exiting; stopping=81db972b9a1d,32805,1732782990425; zookeeper connection closed. 2024-11-28T08:38:16,327 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7091006a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7091006a 2024-11-28T08:38:16,328 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T08:38:16,664 INFO [M:0;81db972b9a1d:36019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T08:38:16,665 INFO [M:0;81db972b9a1d:36019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T08:38:16,665 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T08:38:16,666 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:38:16,666 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:38:16,666 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T08:38:16,666 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:38:16,667 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.98 KB heapSize=190.84 KB 2024-11-28T08:38:16,689 DEBUG [M:0;81db972b9a1d:36019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/709a917c93ea45c88077a65524e985b7 is 82, key is hbase:meta,,1/info:regioninfo/1732783023594/Put/seqid=0 2024-11-28T08:38:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741869_1045 (size=5672) 2024-11-28T08:38:17,097 INFO [M:0;81db972b9a1d:36019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/709a917c93ea45c88077a65524e985b7 2024-11-28T08:38:17,127 DEBUG [M:0;81db972b9a1d:36019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a84bafe5e2674976b371b74fb060399d is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732783025265/Put/seqid=0 2024-11-28T08:38:17,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741870_1046 (size=14069) 2024-11-28T08:38:17,254 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T08:38:17,534 INFO [M:0;81db972b9a1d:36019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.42 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a84bafe5e2674976b371b74fb060399d 2024-11-28T08:38:17,546 INFO [M:0;81db972b9a1d:36019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a84bafe5e2674976b371b74fb060399d 2024-11-28T08:38:17,565 DEBUG [M:0;81db972b9a1d:36019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7135109978a498c8a3b5a9c54dc1158 is 69, key is 81db972b9a1d,32805,1732782990425/rs:state/1732782992921/Put/seqid=0 2024-11-28T08:38:17,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741871_1047 (size=5156) 2024-11-28T08:38:17,972 INFO [M:0;81db972b9a1d:36019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7135109978a498c8a3b5a9c54dc1158 2024-11-28T08:38:17,984 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/709a917c93ea45c88077a65524e985b7 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/709a917c93ea45c88077a65524e985b7 2024-11-28T08:38:17,991 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/709a917c93ea45c88077a65524e985b7, entries=8, sequenceid=317, filesize=5.5 K 2024-11-28T08:38:17,992 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a84bafe5e2674976b371b74fb060399d as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a84bafe5e2674976b371b74fb060399d 2024-11-28T08:38:17,999 INFO [M:0;81db972b9a1d:36019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a84bafe5e2674976b371b74fb060399d 2024-11-28T08:38:17,999 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a84bafe5e2674976b371b74fb060399d, entries=33, sequenceid=317, filesize=13.7 K 2024-11-28T08:38:18,000 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7135109978a498c8a3b5a9c54dc1158 as hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b7135109978a498c8a3b5a9c54dc1158 2024-11-28T08:38:18,007 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33719/user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b7135109978a498c8a3b5a9c54dc1158, entries=1, sequenceid=317, filesize=5.0 K 2024-11-28T08:38:18,008 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.98 KB/166891, heapSize ~190.54 KB/195112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1342ms, sequenceid=317, compaction requested=false 2024-11-28T08:38:18,009 INFO [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T08:38:18,009 DEBUG [M:0;81db972b9a1d:36019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732783096665Disabling compacts and flushes for region at 1732783096665Disabling writes for close at 1732783096666 (+1 ms)Obtaining lock to block concurrent updates at 1732783096667 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732783096667Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166891, getHeapSize=195352, getOffHeapSize=0, getCellsCount=364 at 1732783096668 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732783096670 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732783096671 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732783096689 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732783096689Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732783097110 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732783097127 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732783097127Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732783097547 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732783097564 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732783097564Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ec21d73: reopening flushed file at 1732783097983 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@458cbc9c: reopening flushed file at 1732783097991 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a192cf9: reopening flushed file at 1732783097999 (+8 ms)Finished flush of dataSize ~162.98 KB/166891, heapSize ~190.54 KB/195112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1342ms, sequenceid=317, compaction requested=false at 1732783098008 (+9 ms)Writing region close event to WAL at 1732783098009 (+1 ms)Closed at 1732783098009 2024-11-28T08:38:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38803 is added to blk_1073741830_1006 (size=188103) 2024-11-28T08:38:18,012 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/254a0741-2bda-028c-df42-fa8d56566cfd/MasterData/WALs/81db972b9a1d,36019,1732782989669/81db972b9a1d%2C36019%2C1732782989669.1732782992138 not finished, retry = 0 2024-11-28T08:38:18,114 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T08:38:18,114 INFO [M:0;81db972b9a1d:36019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T08:38:18,114 INFO [M:0;81db972b9a1d:36019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36019 2024-11-28T08:38:18,115 INFO [M:0;81db972b9a1d:36019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T08:38:18,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T08:38:18,271 INFO [M:0;81db972b9a1d:36019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T08:38:18,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36019-0x101809619d40000, quorum=127.0.0.1:57083, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T08:38:18,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc78d53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T08:38:18,280 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T08:38:18,280 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T08:38:18,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T08:38:18,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/hadoop.log.dir/,STOPPED} 2024-11-28T08:38:18,283 WARN [BP-1694812817-172.17.0.3-1732782985901 heartbeating to localhost/127.0.0.1:33719 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T08:38:18,283 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T08:38:18,283 WARN [BP-1694812817-172.17.0.3-1732782985901 heartbeating to localhost/127.0.0.1:33719 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1694812817-172.17.0.3-1732782985901 (Datanode Uuid ed0374b9-ab4e-478d-8623-e28e1fc547d1) service to localhost/127.0.0.1:33719 2024-11-28T08:38:18,283 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T08:38:18,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/data/data1/current/BP-1694812817-172.17.0.3-1732782985901 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T08:38:18,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/cluster_6ec9704d-0cab-75c9-ad08-76d82e43933f/data/data2/current/BP-1694812817-172.17.0.3-1732782985901 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T08:38:18,285 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T08:38:18,296 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@562f0d0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T08:38:18,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T08:38:18,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T08:38:18,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T08:38:18,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/118ea47c-9834-5549-6ce3-377803c88037/hadoop.log.dir/,STOPPED} 2024-11-28T08:38:18,303 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T08:38:18,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down