2024-11-07 12:17:35,554 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-07 12:17:35,604 main DEBUG Took 0.031174 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 12:17:35,605 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 12:17:35,606 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 12:17:35,621 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 12:17:35,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,714 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 12:17:35,759 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,766 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,769 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,771 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,772 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,773 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,773 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,776 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,778 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,779 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,780 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,781 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,782 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,783 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,785 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,786 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:17:35,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,787 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 12:17:35,789 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:17:35,791 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 12:17:35,793 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 12:17:35,793 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 12:17:35,795 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 12:17:35,795 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 12:17:35,811 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 12:17:35,816 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 12:17:35,819 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 12:17:35,819 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 12:17:35,820 main DEBUG createAppenders(={Console}) 2024-11-07 12:17:35,821 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-07 12:17:35,821 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-07 12:17:35,822 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-07 12:17:35,822 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 12:17:35,823 main DEBUG OutputStream closed 2024-11-07 12:17:35,823 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 12:17:35,824 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 12:17:35,824 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-07 12:17:36,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 12:17:36,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 12:17:36,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 12:17:36,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 12:17:36,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 12:17:36,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 12:17:36,128 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 12:17:36,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 12:17:36,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 12:17:36,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 12:17:36,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 12:17:36,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 12:17:36,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 12:17:36,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 12:17:36,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 12:17:36,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 12:17:36,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 12:17:36,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 12:17:36,137 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 12:17:36,138 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-07 12:17:36,144 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 12:17:36,146 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-07T12:17:36,782 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-07 12:17:36,793 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 12:17:36,794 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T12:17:37,022 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db 2024-11-07T12:17:37,104 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T12:17:37,112 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1, deleteOnExit=true 2024-11-07T12:17:37,113 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-07T12:17:37,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/test.cache.data in system properties and HBase conf 2024-11-07T12:17:37,115 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T12:17:37,117 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/hadoop.log.dir in system properties and HBase conf 2024-11-07T12:17:37,121 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T12:17:37,122 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T12:17:37,125 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-07T12:17:37,431 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T12:17:37,761 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T12:17:37,773 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:17:37,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:17:37,785 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T12:17:37,786 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:17:37,786 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T12:17:37,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T12:17:37,792 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:17:37,793 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:17:37,793 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T12:17:37,794 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/nfs.dump.dir in system properties and HBase conf 2024-11-07T12:17:37,794 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/java.io.tmpdir in system properties and HBase conf 2024-11-07T12:17:37,795 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:17:37,795 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T12:17:37,820 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T12:17:39,778 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T12:17:40,004 INFO [Time-limited test {}] log.Log(170): Logging initialized @5843ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T12:17:40,248 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:17:40,418 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:17:40,555 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:17:40,557 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:17:40,559 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T12:17:40,585 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:17:40,606 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1857c056{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:17:40,608 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4803ff98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:17:41,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f422bdf{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/java.io.tmpdir/jetty-localhost-34343-hadoop-hdfs-3_4_1-tests_jar-_-any-7568880058117726567/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:17:41,117 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c457aa{HTTP/1.1, (http/1.1)}{localhost:34343} 2024-11-07T12:17:41,117 INFO [Time-limited test {}] server.Server(415): Started @6957ms 2024-11-07T12:17:42,118 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:17:42,138 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:17:42,158 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:17:42,159 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:17:42,159 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T12:17:42,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@318641df{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:17:42,162 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cf34ac6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:17:42,399 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b30907c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/java.io.tmpdir/jetty-localhost-43193-hadoop-hdfs-3_4_1-tests_jar-_-any-1315886357929586265/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:17:42,407 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2aa1746d{HTTP/1.1, (http/1.1)}{localhost:43193} 2024-11-07T12:17:42,408 INFO [Time-limited test {}] server.Server(415): Started @8248ms 2024-11-07T12:17:42,571 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T12:17:43,726 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/dfs/data/data1/current/BP-1253633137-172.17.0.2-1730981859271/current, will proceed with Du for space computation calculation, 2024-11-07T12:17:43,730 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/dfs/data/data2/current/BP-1253633137-172.17.0.2-1730981859271/current, will proceed with Du for space computation calculation, 2024-11-07T12:17:43,831 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T12:17:43,925 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21d4b12d7281f818 with lease ID 0xf79fd9cdaebd5631: Processing first storage report for DS-2bede43f-0f82-4f3f-a712-4cef2c8ade53 from datanode DatanodeRegistration(127.0.0.1:39577, datanodeUuid=83c6d118-22a1-4447-8a00-68f28783d20d, infoPort=42791, infoSecurePort=0, ipcPort=33513, storageInfo=lv=-57;cid=testClusterID;nsid=816198689;c=1730981859271) 2024-11-07T12:17:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21d4b12d7281f818 with lease ID 0xf79fd9cdaebd5631: from storage DS-2bede43f-0f82-4f3f-a712-4cef2c8ade53 node DatanodeRegistration(127.0.0.1:39577, datanodeUuid=83c6d118-22a1-4447-8a00-68f28783d20d, infoPort=42791, infoSecurePort=0, ipcPort=33513, storageInfo=lv=-57;cid=testClusterID;nsid=816198689;c=1730981859271), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T12:17:43,928 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21d4b12d7281f818 with lease ID 0xf79fd9cdaebd5631: Processing first storage report for DS-95e99409-5d4b-4674-867f-58c6fbdb4545 from datanode DatanodeRegistration(127.0.0.1:39577, datanodeUuid=83c6d118-22a1-4447-8a00-68f28783d20d, infoPort=42791, infoSecurePort=0, ipcPort=33513, storageInfo=lv=-57;cid=testClusterID;nsid=816198689;c=1730981859271) 2024-11-07T12:17:43,928 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21d4b12d7281f818 with lease ID 0xf79fd9cdaebd5631: from storage DS-95e99409-5d4b-4674-867f-58c6fbdb4545 node DatanodeRegistration(127.0.0.1:39577, datanodeUuid=83c6d118-22a1-4447-8a00-68f28783d20d, infoPort=42791, infoSecurePort=0, ipcPort=33513, storageInfo=lv=-57;cid=testClusterID;nsid=816198689;c=1730981859271), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T12:17:44,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db 2024-11-07T12:17:44,337 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/zookeeper_0, clientPort=51230, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T12:17:44,354 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51230 2024-11-07T12:17:44,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:44,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:44,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741825_1001 (size=7) 2024-11-07T12:17:45,439 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc with version=8 2024-11-07T12:17:45,439 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/hbase-staging 2024-11-07T12:17:45,696 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T12:17:46,157 INFO [Time-limited test {}] client.ConnectionUtils(129): master/fbad4c8a32e4:0 server-side Connection retries=9 2024-11-07T12:17:46,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:17:46,200 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:17:46,200 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:17:46,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:17:46,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:17:46,497 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T12:17:46,599 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T12:17:46,612 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T12:17:46,619 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:17:46,675 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 11394 (auto-detected) 2024-11-07T12:17:46,676 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T12:17:46,706 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36333 2024-11-07T12:17:46,739 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:46,742 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:46,782 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36333 connecting to ZooKeeper ensemble=127.0.0.1:51230 2024-11-07T12:17:46,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:363330x0, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:17:46,845 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36333-0x10182aa98860000 connected 2024-11-07T12:17:47,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T12:17:47,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:17:47,014 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:17:47,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36333 2024-11-07T12:17:47,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36333 2024-11-07T12:17:47,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36333 2024-11-07T12:17:47,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36333 2024-11-07T12:17:47,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36333 2024-11-07T12:17:47,116 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc, hbase.cluster.distributed=false 2024-11-07T12:17:47,256 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/fbad4c8a32e4:0 server-side Connection retries=9 2024-11-07T12:17:47,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:17:47,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:17:47,256 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:17:47,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:17:47,257 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:17:47,260 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T12:17:47,263 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:17:47,265 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44527 2024-11-07T12:17:47,267 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T12:17:47,280 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T12:17:47,282 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:47,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:47,291 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:44527 connecting to ZooKeeper ensemble=127.0.0.1:51230 2024-11-07T12:17:47,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:445270x0, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T12:17:47,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:445270x0, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:17:47,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:445270x0, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:17:47,346 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:445270x0, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:17:47,356 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44527-0x10182aa98860001 connected 2024-11-07T12:17:47,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44527 2024-11-07T12:17:47,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44527 2024-11-07T12:17:47,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44527 2024-11-07T12:17:47,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44527 2024-11-07T12:17:47,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44527 2024-11-07T12:17:47,445 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:47,466 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fbad4c8a32e4:36333 2024-11-07T12:17:47,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:17:47,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:17:47,475 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:47,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T12:17:47,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:47,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T12:17:47,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:47,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T12:17:47,531 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T12:17:47,534 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fbad4c8a32e4,36333,1730981865680 from backup master directory 2024-11-07T12:17:47,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:17:47,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:47,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:17:47,544 WARN [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:17:47,544 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:47,547 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T12:17:47,560 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T12:17:47,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741826_1002 (size=42) 2024-11-07T12:17:47,688 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/hbase.id with ID: 0ac2d89b-0245-4725-a687-d6c69c166c14 2024-11-07T12:17:47,775 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:17:47,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:47,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:47,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741827_1003 (size=196) 2024-11-07T12:17:47,949 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:17:47,955 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T12:17:48,013 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:17:48,029 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:17:48,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741828_1004 (size=1189) 2024-11-07T12:17:48,641 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store 2024-11-07T12:17:48,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741829_1005 (size=34) 2024-11-07T12:17:49,136 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T12:17:49,137 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:17:49,138 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:17:49,139 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:17:49,139 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:17:49,139 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:17:49,139 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:17:49,140 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:17:49,140 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T12:17:49,144 WARN [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/.initializing 2024-11-07T12:17:49,144 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/WALs/fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:49,158 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:17:49,189 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fbad4c8a32e4%2C36333%2C1730981865680, suffix=, logDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/WALs/fbad4c8a32e4,36333,1730981865680, archiveDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/oldWALs, maxLogs=10 2024-11-07T12:17:49,222 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/WALs/fbad4c8a32e4,36333,1730981865680/fbad4c8a32e4%2C36333%2C1730981865680.1730981869195, exclude list is [], retry=0 2024-11-07T12:17:49,265 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39577,DS-2bede43f-0f82-4f3f-a712-4cef2c8ade53,DISK] 2024-11-07T12:17:49,272 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T12:17:49,367 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/WALs/fbad4c8a32e4,36333,1730981865680/fbad4c8a32e4%2C36333%2C1730981865680.1730981869195 2024-11-07T12:17:49,374 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42791:42791)] 2024-11-07T12:17:49,375 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:17:49,375 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:17:49,380 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,381 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T12:17:49,646 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:49,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:17:49,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T12:17:49,670 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:49,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:17:49,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T12:17:49,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:49,686 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:17:49,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T12:17:49,696 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:49,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:17:49,706 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,707 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,733 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T12:17:49,758 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:17:49,770 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:17:49,772 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59652132, jitterRate=-0.11111396551132202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T12:17:49,783 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T12:17:49,788 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T12:17:49,865 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127cb425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:17:49,933 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-07T12:17:49,952 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T12:17:49,952 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T12:17:49,955 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T12:17:49,958 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 2 msec 2024-11-07T12:17:49,966 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 7 msec 2024-11-07T12:17:49,966 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T12:17:50,035 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T12:17:50,058 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T12:17:50,062 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-07T12:17:50,067 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T12:17:50,070 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T12:17:50,074 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-07T12:17:50,078 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T12:17:50,095 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T12:17:50,100 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-07T12:17:50,102 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T12:17:50,105 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T12:17:50,142 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T12:17:50,146 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T12:17:50,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:17:50,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:17:50,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:50,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:50,158 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=fbad4c8a32e4,36333,1730981865680, sessionid=0x10182aa98860000, setting cluster-up flag (Was=false) 2024-11-07T12:17:50,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:50,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:50,223 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T12:17:50,230 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:50,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:50,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:17:50,258 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T12:17:50,261 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fbad4c8a32e4,36333,1730981865680 2024-11-07T12:17:50,407 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fbad4c8a32e4:44527 2024-11-07T12:17:50,409 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1008): ClusterId : 0ac2d89b-0245-4725-a687-d6c69c166c14 2024-11-07T12:17:50,414 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T12:17:50,426 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T12:17:50,426 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T12:17:50,432 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T12:17:50,434 DEBUG [RS:0;fbad4c8a32e4:44527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419ce49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:17:50,446 DEBUG [RS:0;fbad4c8a32e4:44527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a1e1a9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fbad4c8a32e4/172.17.0.2:0 2024-11-07T12:17:50,457 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-07T12:17:50,457 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-07T12:17:50,458 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-07T12:17:50,460 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-07T12:17:50,470 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-07T12:17:50,470 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3073): reportForDuty to master=fbad4c8a32e4,36333,1730981865680 with isa=fbad4c8a32e4/172.17.0.2:44527, startcode=1730981867254 2024-11-07T12:17:50,475 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T12:17:50,487 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fbad4c8a32e4,36333,1730981865680 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T12:17:50,492 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fbad4c8a32e4:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:17:50,493 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fbad4c8a32e4:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:17:50,493 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fbad4c8a32e4:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:17:50,494 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fbad4c8a32e4:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:17:50,494 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fbad4c8a32e4:0, corePoolSize=10, maxPoolSize=10 2024-11-07T12:17:50,494 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,494 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fbad4c8a32e4:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:17:50,495 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,504 DEBUG [RS:0;fbad4c8a32e4:44527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T12:17:50,522 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:17:50,523 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-07T12:17:50,535 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:50,535 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T12:17:50,540 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730981900540 2024-11-07T12:17:50,542 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T12:17:50,543 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T12:17:50,547 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T12:17:50,548 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T12:17:50,548 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T12:17:50,548 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T12:17:50,554 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,584 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T12:17:50,586 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T12:17:50,587 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T12:17:50,604 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T12:17:50,613 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T12:17:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741831_1007 (size=1039) 2024-11-07T12:17:50,641 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-07T12:17:50,642 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:17:50,642 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fbad4c8a32e4:0:becomeActiveMaster-HFileCleaner.large.0-1730981870615,5,FailOnTimeoutGroup] 2024-11-07T12:17:50,656 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fbad4c8a32e4:0:becomeActiveMaster-HFileCleaner.small.0-1730981870645,5,FailOnTimeoutGroup] 2024-11-07T12:17:50,657 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,657 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T12:17:50,662 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,663 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,664 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53565, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T12:17:50,674 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36333 {}] master.ServerManager(332): Checking decommissioned status of RegionServer fbad4c8a32e4,44527,1730981867254 2024-11-07T12:17:50,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36333 {}] master.ServerManager(486): Registering regionserver=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:17:50,705 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:17:50,705 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:33401 2024-11-07T12:17:50,705 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-07T12:17:50,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:17:50,712 DEBUG [RS:0;fbad4c8a32e4:44527 {}] zookeeper.ZKUtil(111): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fbad4c8a32e4,44527,1730981867254 2024-11-07T12:17:50,712 WARN [RS:0;fbad4c8a32e4:44527 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:17:50,712 INFO [RS:0;fbad4c8a32e4:44527 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:17:50,712 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254 2024-11-07T12:17:50,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741832_1008 (size=32) 2024-11-07T12:17:50,742 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fbad4c8a32e4,44527,1730981867254] 2024-11-07T12:17:50,783 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-07T12:17:50,801 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T12:17:50,841 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T12:17:50,856 INFO [RS:0;fbad4c8a32e4:44527 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T12:17:50,856 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,858 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-07T12:17:50,884 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,885 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,885 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,885 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,890 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,892 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,892 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fbad4c8a32e4:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:17:50,892 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,892 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,893 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,893 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,893 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fbad4c8a32e4:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:17:50,893 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fbad4c8a32e4:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:17:50,894 DEBUG [RS:0;fbad4c8a32e4:44527 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fbad4c8a32e4:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:17:50,914 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,915 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,915 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,915 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:50,919 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,44527,1730981867254-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:17:51,002 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T12:17:51,008 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,44527,1730981867254-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:17:51,064 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.Replication(204): fbad4c8a32e4,44527,1730981867254 started 2024-11-07T12:17:51,064 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1767): Serving as fbad4c8a32e4,44527,1730981867254, RpcServer on fbad4c8a32e4/172.17.0.2:44527, sessionid=0x10182aa98860001 2024-11-07T12:17:51,080 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T12:17:51,081 DEBUG [RS:0;fbad4c8a32e4:44527 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fbad4c8a32e4,44527,1730981867254 2024-11-07T12:17:51,081 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fbad4c8a32e4,44527,1730981867254' 2024-11-07T12:17:51,081 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T12:17:51,085 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T12:17:51,094 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T12:17:51,095 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T12:17:51,095 DEBUG [RS:0;fbad4c8a32e4:44527 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fbad4c8a32e4,44527,1730981867254 2024-11-07T12:17:51,095 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fbad4c8a32e4,44527,1730981867254' 2024-11-07T12:17:51,095 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T12:17:51,101 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T12:17:51,103 DEBUG [RS:0;fbad4c8a32e4:44527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T12:17:51,103 INFO [RS:0;fbad4c8a32e4:44527 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T12:17:51,103 INFO [RS:0;fbad4c8a32e4:44527 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T12:17:51,135 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:17:51,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:17:51,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:17:51,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:51,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:17:51,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:17:51,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:17:51,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:51,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:17:51,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:17:51,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:17:51,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:17:51,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:17:51,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740 2024-11-07T12:17:51,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740 2024-11-07T12:17:51,216 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T12:17:51,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T12:17:51,236 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:17:51,238 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65763655, jitterRate=-0.020045176148414612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T12:17:51,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T12:17:51,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:17:51,242 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T12:17:51,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T12:17:51,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:17:51,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:17:51,254 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T12:17:51,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T12:17:51,259 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:17:51,259 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-07T12:17:51,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T12:17:51,292 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:17:51,296 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T12:17:51,448 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:52,449 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:53,450 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:54,451 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:55,451 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:56,452 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:56,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T12:17:56,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T12:17:57,453 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:58,453 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:17:59,454 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:00,455 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:01,455 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:02,456 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:03,457 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:04,460 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:05,464 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:06,477 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:07,482 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:08,483 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:09,484 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:10,485 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:11,485 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:12,486 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:13,487 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:14,016 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:18:14,488 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:14,643 ERROR [LeaseRenewer:jenkins@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:33401,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:18:15,489 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:16,489 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:17,490 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:18,491 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:19,496 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:20,500 WARN [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T12:18:21,111 INFO [RS:0;fbad4c8a32e4:44527 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:18:21,141 INFO [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fbad4c8a32e4%2C44527%2C1730981867254, suffix=, logDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254, archiveDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/oldWALs, maxLogs=32 2024-11-07T12:18:21,203 DEBUG [RS:0;fbad4c8a32e4:44527 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254/fbad4c8a32e4%2C44527%2C1730981867254.1730981901145, exclude list is [], retry=0 2024-11-07T12:18:21,221 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39577,DS-2bede43f-0f82-4f3f-a712-4cef2c8ade53,DISK] 2024-11-07T12:18:21,237 INFO [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254/fbad4c8a32e4%2C44527%2C1730981867254.1730981901145 2024-11-07T12:18:21,237 DEBUG [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42791:42791)] 2024-11-07T12:18:21,504 DEBUG [fbad4c8a32e4:36333 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T12:18:21,510 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:21,517 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fbad4c8a32e4,44527,1730981867254, state=OPENING 2024-11-07T12:18:21,533 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T12:18:21,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:18:21,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:18:21,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:18:21,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:18:21,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:21,738 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:21,741 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T12:18:21,749 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38340, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T12:18:21,769 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-07T12:18:21,769 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:18:21,770 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T12:18:21,779 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fbad4c8a32e4%2C44527%2C1730981867254.meta, suffix=.meta, logDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254, archiveDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/oldWALs, maxLogs=32 2024-11-07T12:18:21,808 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254/fbad4c8a32e4%2C44527%2C1730981867254.meta.1730981901782.meta, exclude list is [], retry=0 2024-11-07T12:18:21,814 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39577,DS-2bede43f-0f82-4f3f-a712-4cef2c8ade53,DISK] 2024-11-07T12:18:21,826 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254/fbad4c8a32e4%2C44527%2C1730981867254.meta.1730981901782.meta 2024-11-07T12:18:21,832 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42791:42791)] 2024-11-07T12:18:21,833 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:21,846 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T12:18:21,944 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T12:18:21,951 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T12:18:21,957 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T12:18:21,958 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:21,959 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-07T12:18:21,959 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-07T12:18:21,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:18:21,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:18:21,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:21,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:18:21,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:18:21,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:18:21,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:21,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:18:21,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:18:21,974 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:18:21,974 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:21,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:18:21,982 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740 2024-11-07T12:18:21,987 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740 2024-11-07T12:18:21,993 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T12:18:21,999 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T12:18:22,002 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72881596, jitterRate=0.08602041006088257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T12:18:22,004 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T12:18:22,016 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730981901730 2024-11-07T12:18:22,035 DEBUG [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T12:18:22,036 INFO [RS_OPEN_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-07T12:18:22,038 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:22,041 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fbad4c8a32e4,44527,1730981867254, state=OPEN 2024-11-07T12:18:22,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:18:22,045 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:18:22,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:18:22,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:18:22,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T12:18:22,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=fbad4c8a32e4,44527,1730981867254 in 496 msec 2024-11-07T12:18:22,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T12:18:22,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.7900 sec 2024-11-07T12:18:22,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 31.7290 sec 2024-11-07T12:18:22,076 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730981902076, completionTime=-1 2024-11-07T12:18:22,077 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T12:18:22,077 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-07T12:18:22,135 DEBUG [hconnection-0x5ef5fe98-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:22,138 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38346, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:22,152 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-07T12:18:22,153 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730981962153 2024-11-07T12:18:22,153 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730982022153 2024-11-07T12:18:22,153 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 76 msec 2024-11-07T12:18:22,206 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,36333,1730981865680-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:18:22,207 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,36333,1730981865680-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:18:22,207 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,36333,1730981865680-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:18:22,210 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fbad4c8a32e4:36333, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:18:22,211 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T12:18:22,220 DEBUG [master/fbad4c8a32e4:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-07T12:18:22,223 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-07T12:18:22,226 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T12:18:22,238 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-07T12:18:22,243 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:22,245 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:22,249 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:22,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741835_1011 (size=358) 2024-11-07T12:18:22,315 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 69f29002b8ac38e61dd30e67f2cf97a7, NAME => 'hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:22,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741836_1012 (size=42) 2024-11-07T12:18:22,342 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:22,342 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 69f29002b8ac38e61dd30e67f2cf97a7, disabling compactions & flushes 2024-11-07T12:18:22,342 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,342 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,342 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. after waiting 0 ms 2024-11-07T12:18:22,342 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,342 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,343 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 69f29002b8ac38e61dd30e67f2cf97a7: 2024-11-07T12:18:22,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:22,355 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1730981902348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981902348"}]},"ts":"1730981902348"} 2024-11-07T12:18:22,395 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:22,398 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:22,402 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981902398"}]},"ts":"1730981902398"} 2024-11-07T12:18:22,409 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-07T12:18:22,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=69f29002b8ac38e61dd30e67f2cf97a7, ASSIGN}] 2024-11-07T12:18:22,422 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=69f29002b8ac38e61dd30e67f2cf97a7, ASSIGN 2024-11-07T12:18:22,424 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=69f29002b8ac38e61dd30e67f2cf97a7, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:22,575 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=69f29002b8ac38e61dd30e67f2cf97a7, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:22,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 69f29002b8ac38e61dd30e67f2cf97a7, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:22,741 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:22,756 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 69f29002b8ac38e61dd30e67f2cf97a7, NAME => 'hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:22,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:22,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,763 INFO [StoreOpener-69f29002b8ac38e61dd30e67f2cf97a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,773 INFO [StoreOpener-69f29002b8ac38e61dd30e67f2cf97a7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69f29002b8ac38e61dd30e67f2cf97a7 columnFamilyName info 2024-11-07T12:18:22,773 DEBUG [StoreOpener-69f29002b8ac38e61dd30e67f2cf97a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:22,781 INFO [StoreOpener-69f29002b8ac38e61dd30e67f2cf97a7-1 {}] regionserver.HStore(327): Store=69f29002b8ac38e61dd30e67f2cf97a7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:22,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,793 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:18:22,823 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:22,827 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 69f29002b8ac38e61dd30e67f2cf97a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59795600, jitterRate=-0.10897612571716309}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:22,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 69f29002b8ac38e61dd30e67f2cf97a7: 2024-11-07T12:18:22,836 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7., pid=6, masterSystemTime=1730981902741 2024-11-07T12:18:22,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,841 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:18:22,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=69f29002b8ac38e61dd30e67f2cf97a7, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:22,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T12:18:22,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 69f29002b8ac38e61dd30e67f2cf97a7, server=fbad4c8a32e4,44527,1730981867254 in 280 msec 2024-11-07T12:18:22,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T12:18:22,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=69f29002b8ac38e61dd30e67f2cf97a7, ASSIGN in 458 msec 2024-11-07T12:18:22,885 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:22,885 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981902885"}]},"ts":"1730981902885"} 2024-11-07T12:18:22,889 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-07T12:18:22,895 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:22,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 668 msec 2024-11-07T12:18:22,943 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-07T12:18:22,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-07T12:18:22,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:18:22,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:18:22,999 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-07T12:18:23,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T12:18:23,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 73 msec 2024-11-07T12:18:23,086 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-07T12:18:23,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T12:18:23,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 40 msec 2024-11-07T12:18:23,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-07T12:18:23,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-07T12:18:23,148 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 35.604sec 2024-11-07T12:18:23,150 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T12:18:23,152 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T12:18:23,155 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T12:18:23,155 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T12:18:23,156 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T12:18:23,157 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,36333,1730981865680-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:18:23,158 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,36333,1730981865680-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T12:18:23,188 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-07T12:18:23,190 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T12:18:23,190 INFO [master/fbad4c8a32e4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fbad4c8a32e4,36333,1730981865680-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:18:23,203 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x551b3f6b to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e633ce7 2024-11-07T12:18:23,210 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-07T12:18:23,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5529b0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:23,239 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T12:18:23,240 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T12:18:23,266 DEBUG [hconnection-0x7939c77a-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:23,279 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38356, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:23,291 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=fbad4c8a32e4,36333,1730981865680 2024-11-07T12:18:23,307 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=217, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=887, ProcessCount=11, AvailableMemoryMB=2611 2024-11-07T12:18:23,342 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x721ecdf9 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@271c70ae 2024-11-07T12:18:23,367 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x721ecdf9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f10a793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:23,372 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x769af929 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32476e54 2024-11-07T12:18:23,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8b4499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:23,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:18:23,420 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33672, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:18:23,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T12:18:23,436 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:23,436 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:23,438 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T12:18:23,440 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33682, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T12:18:23,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:23,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-07T12:18:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T12:18:23,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741837_1013 (size=437) 2024-11-07T12:18:23,481 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0e1920f7313ec7776012f1bb237576bf, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:23,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741838_1014 (size=88) 2024-11-07T12:18:23,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:23,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 0e1920f7313ec7776012f1bb237576bf, disabling compactions & flushes 2024-11-07T12:18:23,508 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. after waiting 0 ms 2024-11-07T12:18:23,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,509 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,509 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0e1920f7313ec7776012f1bb237576bf: 2024-11-07T12:18:23,518 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:23,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1730981903518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981903518"}]},"ts":"1730981903518"} 2024-11-07T12:18:23,523 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:23,525 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:23,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981903525"}]},"ts":"1730981903525"} 2024-11-07T12:18:23,529 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:23,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=0e1920f7313ec7776012f1bb237576bf, ASSIGN}] 2024-11-07T12:18:23,537 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=0e1920f7313ec7776012f1bb237576bf, ASSIGN 2024-11-07T12:18:23,539 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=0e1920f7313ec7776012f1bb237576bf, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:23,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=0e1920f7313ec7776012f1bb237576bf, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:23,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 0e1920f7313ec7776012f1bb237576bf, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:23,849 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:23,856 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,856 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 0e1920f7313ec7776012f1bb237576bf, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:23,856 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,857 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:23,857 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,857 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,859 INFO [StoreOpener-0e1920f7313ec7776012f1bb237576bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,862 INFO [StoreOpener-0e1920f7313ec7776012f1bb237576bf-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1920f7313ec7776012f1bb237576bf columnFamilyName testFamily 2024-11-07T12:18:23,862 DEBUG [StoreOpener-0e1920f7313ec7776012f1bb237576bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:23,863 INFO [StoreOpener-0e1920f7313ec7776012f1bb237576bf-1 {}] regionserver.HStore(327): Store=0e1920f7313ec7776012f1bb237576bf/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:23,864 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,865 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,868 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:18:23,879 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:23,881 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 0e1920f7313ec7776012f1bb237576bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74559132, jitterRate=0.11101764440536499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:23,882 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 0e1920f7313ec7776012f1bb237576bf: 2024-11-07T12:18:23,884 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf., pid=11, masterSystemTime=1730981903849 2024-11-07T12:18:23,888 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,889 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:18:23,889 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=0e1920f7313ec7776012f1bb237576bf, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:23,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-07T12:18:23,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 0e1920f7313ec7776012f1bb237576bf, server=fbad4c8a32e4,44527,1730981867254 in 199 msec 2024-11-07T12:18:23,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T12:18:23,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=0e1920f7313ec7776012f1bb237576bf, ASSIGN in 363 msec 2024-11-07T12:18:23,902 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:23,903 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981903902"}]},"ts":"1730981903902"} 2024-11-07T12:18:23,906 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:23,911 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:23,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 479 msec 2024-11-07T12:18:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T12:18:24,457 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-07T12:18:24,457 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:24,459 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:24,464 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:24,464 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:24,464 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-07T12:18:24,468 DEBUG [hconnection-0x736f7eab-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:24,471 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38368, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:24,496 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:24,504 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T12:18:24,518 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:18:24,520 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x721ecdf9 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:18:24,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:24,528 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38378, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:24,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:18:24,540 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T12:18:24,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581451 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:24,543 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T12:18:24,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581451 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:24,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970165 2024-11-07T12:18:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38378 deadline: 1730981907542, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:24,569 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581451 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:18:24,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T12:18:24,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581452 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:24,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T12:18:24,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:24,576 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:24,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581453 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:24,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970163 2024-11-07T12:18:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38378 deadline: 1730981907575, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:24,578 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581453 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:18:24,579 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:24,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581454 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:24,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1672048081345970162 2024-11-07T12:18:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38378 deadline: 1730981907579, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:24,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581455 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:24,581 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581454 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:18:24,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1672048081345970161 2024-11-07T12:18:24,585 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:38378 deadline: 1730981907582, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:24,586 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581455 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:18:24,588 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:24,589 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x769af929 to 127.0.0.1:51230 2024-11-07T12:18:24,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:24,590 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:18:24,590 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:18:24,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:24,596 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x721ecdf9 to 127.0.0.1:51230 2024-11-07T12:18:24,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=230 (was 217) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51230@0x721ecdf9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51230@0x769af929-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51230@0x769af929 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$730/0x00007f4cf0b44180.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51230@0x721ecdf9 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$730/0x00007f4cf0b44180.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x7939c77a-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51230@0x769af929-SendThread(127.0.0.1:51230) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51230@0x721ecdf9-SendThread(127.0.0.1:51230) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1861550235_22 at /127.0.0.1:56972 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=856 (was 887), ProcessCount=11 (was 11), AvailableMemoryMB=2438 (was 2611) 2024-11-07T12:18:24,621 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=230, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=856, ProcessCount=11, AvailableMemoryMB=2438 2024-11-07T12:18:24,623 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ed48db9 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a69ab7 2024-11-07T12:18:24,627 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x5ed48db9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa43d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:24,628 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x033e9983 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@775ac82d 2024-11-07T12:18:24,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d581e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:24,635 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:24,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T12:18:24,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:24,639 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:24,639 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-07T12:18:24,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T12:18:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741839_1015 (size=423) 2024-11-07T12:18:25,058 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7db5f0ffc15ecc581283f92d21730cb9, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741840_1016 (size=74) 2024-11-07T12:18:25,469 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:25,469 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 7db5f0ffc15ecc581283f92d21730cb9, disabling compactions & flushes 2024-11-07T12:18:25,469 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,469 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,469 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. after waiting 0 ms 2024-11-07T12:18:25,469 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,469 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,469 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7db5f0ffc15ecc581283f92d21730cb9: 2024-11-07T12:18:25,472 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:25,472 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1730981905472"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981905472"}]},"ts":"1730981905472"} 2024-11-07T12:18:25,476 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:25,478 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:25,478 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981905478"}]},"ts":"1730981905478"} 2024-11-07T12:18:25,481 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:25,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7db5f0ffc15ecc581283f92d21730cb9, ASSIGN}] 2024-11-07T12:18:25,487 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7db5f0ffc15ecc581283f92d21730cb9, ASSIGN 2024-11-07T12:18:25,488 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7db5f0ffc15ecc581283f92d21730cb9, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:25,639 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=7db5f0ffc15ecc581283f92d21730cb9, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:25,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 7db5f0ffc15ecc581283f92d21730cb9, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T12:18:25,796 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:25,801 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => 7db5f0ffc15ecc581283f92d21730cb9, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:25,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:25,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for 7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for 7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,805 INFO [StoreOpener-7db5f0ffc15ecc581283f92d21730cb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,807 INFO [StoreOpener-7db5f0ffc15ecc581283f92d21730cb9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7db5f0ffc15ecc581283f92d21730cb9 columnFamilyName testFamily 2024-11-07T12:18:25,807 DEBUG [StoreOpener-7db5f0ffc15ecc581283f92d21730cb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:25,808 INFO [StoreOpener-7db5f0ffc15ecc581283f92d21730cb9-1 {}] regionserver.HStore(327): Store=7db5f0ffc15ecc581283f92d21730cb9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:25,809 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for 7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:18:25,817 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:25,817 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened 7db5f0ffc15ecc581283f92d21730cb9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71300699, jitterRate=0.06246320903301239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:25,818 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for 7db5f0ffc15ecc581283f92d21730cb9: 2024-11-07T12:18:25,820 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9., pid=14, masterSystemTime=1730981905795 2024-11-07T12:18:25,822 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,823 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:18:25,823 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=7db5f0ffc15ecc581283f92d21730cb9, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:25,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T12:18:25,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 7db5f0ffc15ecc581283f92d21730cb9, server=fbad4c8a32e4,44527,1730981867254 in 184 msec 2024-11-07T12:18:25,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-07T12:18:25,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7db5f0ffc15ecc581283f92d21730cb9, ASSIGN in 344 msec 2024-11-07T12:18:25,833 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:25,834 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981905834"}]},"ts":"1730981905834"} 2024-11-07T12:18:25,836 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:25,840 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:25,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.2050 sec 2024-11-07T12:18:26,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T12:18:26,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:26,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T12:18:26,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:26,583 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-07T12:18:26,583 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:26,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:18:26,584 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T12:18:26,585 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T12:18:26,585 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:26,794 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-07T12:18:26,795 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T12:18:26,796 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-07T12:18:26,797 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-07T12:18:27,512 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:18:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T12:18:27,646 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-07T12:18:27,646 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:27,646 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:27,651 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:27,651 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:27,651 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T12:18:27,654 DEBUG [hconnection-0x741592e1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:27,656 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38390, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:27,670 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:27,673 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:18:27,674 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:18:34,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T12:18:34,707 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:18:36,682 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:18:36,683 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T12:18:36.683Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11010: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1976ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:18:36,686 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9013 2024-11-07T12:18:36,687 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x033e9983 to 127.0.0.1:51230 2024-11-07T12:18:36,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:36,687 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:18:36,687 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:18:36,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:36,688 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ed48db9 to 127.0.0.1:51230 2024-11-07T12:18:36,702 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=220 (was 230), OpenFileDescriptor=450 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=772 (was 856), ProcessCount=11 (was 11), AvailableMemoryMB=2149 (was 2438) 2024-11-07T12:18:36,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=220, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=772, ProcessCount=11, AvailableMemoryMB=2149 2024-11-07T12:18:36,717 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d570d84 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@378ac15a 2024-11-07T12:18:36,722 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x1d570d84 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fc1c6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:36,723 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e238057 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@236bbfa4 2024-11-07T12:18:36,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5ad60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:36,732 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:36,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T12:18:36,740 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:36,740 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:36,741 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-07T12:18:36,742 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T12:18:36,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741841_1017 (size=421) 2024-11-07T12:18:36,757 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2afefd66dadc9c5d4483629ec062e7ed, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:36,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741842_1018 (size=72) 2024-11-07T12:18:36,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:36,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2afefd66dadc9c5d4483629ec062e7ed, disabling compactions & flushes 2024-11-07T12:18:36,776 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:36,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:36,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. after waiting 0 ms 2024-11-07T12:18:36,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:36,776 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:36,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2afefd66dadc9c5d4483629ec062e7ed: 2024-11-07T12:18:36,778 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:36,778 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730981916778"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981916778"}]},"ts":"1730981916778"} 2024-11-07T12:18:36,784 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:36,786 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:36,786 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981916786"}]},"ts":"1730981916786"} 2024-11-07T12:18:36,789 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:36,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=2afefd66dadc9c5d4483629ec062e7ed, ASSIGN}] 2024-11-07T12:18:36,797 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=2afefd66dadc9c5d4483629ec062e7ed, ASSIGN 2024-11-07T12:18:36,799 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=2afefd66dadc9c5d4483629ec062e7ed, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:36,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=2afefd66dadc9c5d4483629ec062e7ed, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:36,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 2afefd66dadc9c5d4483629ec062e7ed, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:37,108 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:37,120 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:37,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 2afefd66dadc9c5d4483629ec062e7ed, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:37,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:37,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,124 INFO [StoreOpener-2afefd66dadc9c5d4483629ec062e7ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,130 INFO [StoreOpener-2afefd66dadc9c5d4483629ec062e7ed-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2afefd66dadc9c5d4483629ec062e7ed columnFamilyName testFamily 2024-11-07T12:18:37,130 DEBUG [StoreOpener-2afefd66dadc9c5d4483629ec062e7ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:37,133 INFO [StoreOpener-2afefd66dadc9c5d4483629ec062e7ed-1 {}] regionserver.HStore(327): Store=2afefd66dadc9c5d4483629ec062e7ed/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:37,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,137 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:18:37,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:37,150 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened 2afefd66dadc9c5d4483629ec062e7ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65780503, jitterRate=-0.019794121384620667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:37,151 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 2afefd66dadc9c5d4483629ec062e7ed: 2024-11-07T12:18:37,153 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed., pid=17, masterSystemTime=1730981917107 2024-11-07T12:18:37,157 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:37,157 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:18:37,158 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=2afefd66dadc9c5d4483629ec062e7ed, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:37,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-07T12:18:37,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 2afefd66dadc9c5d4483629ec062e7ed, server=fbad4c8a32e4,44527,1730981867254 in 207 msec 2024-11-07T12:18:37,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T12:18:37,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=2afefd66dadc9c5d4483629ec062e7ed, ASSIGN in 371 msec 2024-11-07T12:18:37,172 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:37,172 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981917172"}]},"ts":"1730981917172"} 2024-11-07T12:18:37,175 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:37,180 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:37,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 447 msec 2024-11-07T12:18:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T12:18:37,746 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-07T12:18:37,746 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:37,747 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:37,752 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:37,753 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:37,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T12:18:37,756 DEBUG [hconnection-0x401a0c43-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:37,764 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54508, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:37,778 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:37,782 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:18:37,782 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:18:37,784 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x1d570d84 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:18:37,784 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x1d570d84 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:37,786 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54510, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:18:37,790 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:18:37,790 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581467 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:37,791 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:18:37,791 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581467 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:37,793 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T12:18:38,799 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-07T12:18:41,208 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:18:42,648 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:18:44,016 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:18:46,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T12:18:46,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:46,794 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9002 2024-11-07T12:18:46,794 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e238057 to 127.0.0.1:51230 2024-11-07T12:18:46,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:46,794 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581467 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:18:46,794 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:18:46,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970149' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T12:18:46,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:54510 deadline: 1730981932793, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970149' does not exist 2024-11-07T12:18:46,795 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:18:46,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:46,795 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d570d84 to 127.0.0.1:51230 2024-11-07T12:18:46,800 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to fbad4c8a32e4,44527,1730981867254 for closing scanner id = -1672048081345970149 for 2afefd66dadc9c5d4483629ec062e7ed of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970149' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:18:46,810 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=220 (was 220), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 772), ProcessCount=11 (was 11), AvailableMemoryMB=1988 (was 2149) 2024-11-07T12:18:46,822 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=220, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=1988 2024-11-07T12:18:46,825 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ff66c6c to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@701b8b6a 2024-11-07T12:18:46,835 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x1ff66c6c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5f94f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:46,837 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x463e8dc8 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3124aff5 2024-11-07T12:18:46,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1d20b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:46,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T12:18:46,856 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:46,856 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:46,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-07T12:18:46,858 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T12:18:46,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741843_1019 (size=432) 2024-11-07T12:18:46,898 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b6063fc2dd91d0f4fc20bca211b18d17, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:46,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741844_1020 (size=83) 2024-11-07T12:18:46,914 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:46,914 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing b6063fc2dd91d0f4fc20bca211b18d17, disabling compactions & flushes 2024-11-07T12:18:46,914 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:46,914 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:46,914 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. after waiting 0 ms 2024-11-07T12:18:46,914 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:46,914 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:46,914 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for b6063fc2dd91d0f4fc20bca211b18d17: 2024-11-07T12:18:46,916 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:46,917 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1730981926916"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981926916"}]},"ts":"1730981926916"} 2024-11-07T12:18:46,920 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:46,922 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:46,922 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981926922"}]},"ts":"1730981926922"} 2024-11-07T12:18:46,925 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:46,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=b6063fc2dd91d0f4fc20bca211b18d17, ASSIGN}] 2024-11-07T12:18:46,937 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=b6063fc2dd91d0f4fc20bca211b18d17, ASSIGN 2024-11-07T12:18:46,939 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=b6063fc2dd91d0f4fc20bca211b18d17, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:47,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=b6063fc2dd91d0f4fc20bca211b18d17, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:47,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure b6063fc2dd91d0f4fc20bca211b18d17, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:47,249 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:47,259 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:47,259 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => b6063fc2dd91d0f4fc20bca211b18d17, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:47,260 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,260 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:47,260 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,260 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,263 INFO [StoreOpener-b6063fc2dd91d0f4fc20bca211b18d17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,267 INFO [StoreOpener-b6063fc2dd91d0f4fc20bca211b18d17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6063fc2dd91d0f4fc20bca211b18d17 columnFamilyName testFamily 2024-11-07T12:18:47,267 DEBUG [StoreOpener-b6063fc2dd91d0f4fc20bca211b18d17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:47,271 INFO [StoreOpener-b6063fc2dd91d0f4fc20bca211b18d17-1 {}] regionserver.HStore(327): Store=b6063fc2dd91d0f4fc20bca211b18d17/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:47,273 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,274 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,292 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:18:47,293 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:18:47,300 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:47,301 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened b6063fc2dd91d0f4fc20bca211b18d17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59663983, jitterRate=-0.11093737185001373}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:47,302 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for b6063fc2dd91d0f4fc20bca211b18d17: 2024-11-07T12:18:47,304 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17., pid=20, masterSystemTime=1730981927249 2024-11-07T12:18:47,307 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:47,307 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:18:47,308 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=b6063fc2dd91d0f4fc20bca211b18d17, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:47,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T12:18:47,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure b6063fc2dd91d0f4fc20bca211b18d17, server=fbad4c8a32e4,44527,1730981867254 in 217 msec 2024-11-07T12:18:47,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-07T12:18:47,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=b6063fc2dd91d0f4fc20bca211b18d17, ASSIGN in 381 msec 2024-11-07T12:18:47,322 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:47,322 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981927322"}]},"ts":"1730981927322"} 2024-11-07T12:18:47,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:47,331 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:47,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 480 msec 2024-11-07T12:18:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T12:18:47,863 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-07T12:18:47,863 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:47,864 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:47,878 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:47,879 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:47,879 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-07T12:18:47,882 DEBUG [hconnection-0x5fb8d58f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:47,887 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50166, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:47,904 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:47,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T12:18:47,920 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T12:18:47,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581471 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:47,923 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T12:18:47,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581471 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:47,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970145 2024-11-07T12:18:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:50166 deadline: 1730981928924, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:47,927 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:18:47,930 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T12:18:47,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581472 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:47,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T12:18:47,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:47,939 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:47,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581473 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:47,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970143 2024-11-07T12:18:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:50166 deadline: 1730981928940, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:47,943 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:18:47,945 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:47,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581474 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:47,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1672048081345970142 2024-11-07T12:18:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:50166 deadline: 1730981928946, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:47,949 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:18:47,952 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:47,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581475 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:47,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1672048081345970141 2024-11-07T12:18:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:50166 deadline: 1730981928953, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:18:47,954 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:18:47,957 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:18:47,958 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x463e8dc8 to 127.0.0.1:51230 2024-11-07T12:18:47,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:47,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:18:47,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:18:47,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:47,958 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ff66c6c to 127.0.0.1:51230 2024-11-07T12:18:47,975 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=226 (was 220) - Thread LEAK? -, OpenFileDescriptor=454 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=1955 (was 1988) 2024-11-07T12:18:47,990 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=1951 2024-11-07T12:18:47,992 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7bdca310 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13e42a18 2024-11-07T12:18:47,997 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x7bdca310 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37777a88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:48,000 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f750215 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@685a0b63 2024-11-07T12:18:48,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7680e15b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:48,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T12:18:48,034 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:48,034 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:48,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-07T12:18:48,036 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T12:18:48,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741845_1021 (size=426) 2024-11-07T12:18:48,481 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2c64a0e802a548485c4a2b86bcd23e46, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:48,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741846_1022 (size=77) 2024-11-07T12:18:48,500 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:48,500 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2c64a0e802a548485c4a2b86bcd23e46, disabling compactions & flushes 2024-11-07T12:18:48,500 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,500 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,500 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. after waiting 0 ms 2024-11-07T12:18:48,500 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,500 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,500 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2c64a0e802a548485c4a2b86bcd23e46: 2024-11-07T12:18:48,503 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:48,503 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730981928503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981928503"}]},"ts":"1730981928503"} 2024-11-07T12:18:48,507 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:48,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:48,509 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981928509"}]},"ts":"1730981928509"} 2024-11-07T12:18:48,513 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:48,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=2c64a0e802a548485c4a2b86bcd23e46, ASSIGN}] 2024-11-07T12:18:48,520 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=2c64a0e802a548485c4a2b86bcd23e46, ASSIGN 2024-11-07T12:18:48,521 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=2c64a0e802a548485c4a2b86bcd23e46, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:48,672 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=2c64a0e802a548485c4a2b86bcd23e46, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:48,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 2c64a0e802a548485c4a2b86bcd23e46, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:48,834 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:48,843 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,843 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => 2c64a0e802a548485c4a2b86bcd23e46, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:48,845 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,845 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:48,845 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for 2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,845 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for 2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,856 INFO [StoreOpener-2c64a0e802a548485c4a2b86bcd23e46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,864 INFO [StoreOpener-2c64a0e802a548485c4a2b86bcd23e46-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c64a0e802a548485c4a2b86bcd23e46 columnFamilyName testFamily 2024-11-07T12:18:48,865 DEBUG [StoreOpener-2c64a0e802a548485c4a2b86bcd23e46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:48,866 INFO [StoreOpener-2c64a0e802a548485c4a2b86bcd23e46-1 {}] regionserver.HStore(327): Store=2c64a0e802a548485c4a2b86bcd23e46/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:48,868 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,869 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,874 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for 2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:18:48,893 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:48,895 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened 2c64a0e802a548485c4a2b86bcd23e46; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60249138, jitterRate=-0.10221788287162781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:48,896 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for 2c64a0e802a548485c4a2b86bcd23e46: 2024-11-07T12:18:48,898 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46., pid=23, masterSystemTime=1730981928834 2024-11-07T12:18:48,901 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,902 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:18:48,902 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=2c64a0e802a548485c4a2b86bcd23e46, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:48,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-07T12:18:48,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 2c64a0e802a548485c4a2b86bcd23e46, server=fbad4c8a32e4,44527,1730981867254 in 230 msec 2024-11-07T12:18:48,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T12:18:48,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=2c64a0e802a548485c4a2b86bcd23e46, ASSIGN in 393 msec 2024-11-07T12:18:48,915 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:48,916 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981928915"}]},"ts":"1730981928915"} 2024-11-07T12:18:48,919 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:48,923 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:48,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 894 msec 2024-11-07T12:18:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T12:18:49,046 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-07T12:18:49,046 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:49,046 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:49,067 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:49,068 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:49,068 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T12:18:49,083 DEBUG [hconnection-0x4ae6d053-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:49,100 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:49,126 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:49,146 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:18:49,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:18:49,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T12:18:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T12:18:50,801 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-07T12:18:50,802 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-07T12:18:51,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:18:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:50180 deadline: 1730981932176, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:18:51,190 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:18:51,191 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581480 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:18:51,192 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2045 2024-11-07T12:18:51,192 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f750215 to 127.0.0.1:51230 2024-11-07T12:18:51,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:51,192 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:18:51,192 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:18:51,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:51,193 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7bdca310 to 127.0.0.1:51230 2024-11-07T12:18:51,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=227 (was 226) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=714 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=1864 (was 1951) 2024-11-07T12:18:51,212 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=714, ProcessCount=11, AvailableMemoryMB=1864 2024-11-07T12:18:51,214 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bdf7dc5 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ca23f97 2024-11-07T12:18:51,217 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x6bdf7dc5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237468a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:51,218 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:33401,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:18:51,219 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a8f66ab to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@715c166f 2024-11-07T12:18:51,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db532f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:51,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T12:18:51,228 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:51,229 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:51,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-07T12:18:51,230 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T12:18:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741847_1023 (size=427) 2024-11-07T12:18:51,642 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6befa1d405285dfda10661543d4825a3, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:51,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741848_1024 (size=78) 2024-11-07T12:18:52,051 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:52,052 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 6befa1d405285dfda10661543d4825a3, disabling compactions & flushes 2024-11-07T12:18:52,052 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,052 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,052 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. after waiting 0 ms 2024-11-07T12:18:52,052 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,052 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,052 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6befa1d405285dfda10661543d4825a3: 2024-11-07T12:18:52,054 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:52,054 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1730981932054"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981932054"}]},"ts":"1730981932054"} 2024-11-07T12:18:52,056 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:52,058 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:52,058 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981932058"}]},"ts":"1730981932058"} 2024-11-07T12:18:52,060 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:52,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6befa1d405285dfda10661543d4825a3, ASSIGN}] 2024-11-07T12:18:52,065 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6befa1d405285dfda10661543d4825a3, ASSIGN 2024-11-07T12:18:52,066 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6befa1d405285dfda10661543d4825a3, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:52,217 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=6befa1d405285dfda10661543d4825a3, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:52,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 6befa1d405285dfda10661543d4825a3, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T12:18:52,296 ERROR [LeaseRenewer:jenkins@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:33401,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:18:52,372 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:52,378 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,378 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => 6befa1d405285dfda10661543d4825a3, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:52,378 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,378 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:52,378 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for 6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,379 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for 6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,381 INFO [StoreOpener-6befa1d405285dfda10661543d4825a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,382 INFO [StoreOpener-6befa1d405285dfda10661543d4825a3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6befa1d405285dfda10661543d4825a3 columnFamilyName testFamily 2024-11-07T12:18:52,383 DEBUG [StoreOpener-6befa1d405285dfda10661543d4825a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:52,383 INFO [StoreOpener-6befa1d405285dfda10661543d4825a3-1 {}] regionserver.HStore(327): Store=6befa1d405285dfda10661543d4825a3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:52,384 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,385 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,387 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for 6befa1d405285dfda10661543d4825a3 2024-11-07T12:18:52,391 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:52,391 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened 6befa1d405285dfda10661543d4825a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72640179, jitterRate=0.08242301642894745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:52,392 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for 6befa1d405285dfda10661543d4825a3: 2024-11-07T12:18:52,393 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3., pid=26, masterSystemTime=1730981932372 2024-11-07T12:18:52,396 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,396 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:18:52,397 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=6befa1d405285dfda10661543d4825a3, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:52,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T12:18:52,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 6befa1d405285dfda10661543d4825a3, server=fbad4c8a32e4,44527,1730981867254 in 179 msec 2024-11-07T12:18:52,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-07T12:18:52,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=6befa1d405285dfda10661543d4825a3, ASSIGN in 340 msec 2024-11-07T12:18:52,407 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:52,407 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981932407"}]},"ts":"1730981932407"} 2024-11-07T12:18:52,409 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:52,412 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:52,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 1.1870 sec 2024-11-07T12:18:52,762 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:18:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T12:18:54,235 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-07T12:18:54,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:54,235 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:54,240 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:54,240 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:54,240 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T12:18:54,242 DEBUG [hconnection-0x3be43f79-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:54,244 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52160, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:54,255 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:54,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:18:54,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:18:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T12:18:56,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:18:56,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T12:18:56,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:56,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T12:18:56,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:56,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T12:18:56,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:18:56,803 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-07T12:18:57,263 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:18:57,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T12:18:57.263Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=990ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:18:57,264 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2024-11-07T12:18:57,264 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a8f66ab to 127.0.0.1:51230 2024-11-07T12:18:57,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:57,265 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:18:57,265 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:18:57,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:18:57,265 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bdf7dc5 to 127.0.0.1:51230 2024-11-07T12:18:57,274 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=223 (was 227), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=665 (was 714), ProcessCount=11 (was 11), AvailableMemoryMB=1814 (was 1864) 2024-11-07T12:18:57,283 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=223, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=665, ProcessCount=11, AvailableMemoryMB=1814 2024-11-07T12:18:57,285 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29be1711 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41b6a52d 2024-11-07T12:18:57,289 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x29be1711 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee92c31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:57,291 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62003296 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@479d10ee 2024-11-07T12:18:57,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec29193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:18:57,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:18:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T12:18:57,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:18:57,301 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:57,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-07T12:18:57,302 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:18:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T12:18:57,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741849_1025 (size=425) 2024-11-07T12:18:57,718 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b472beaddbb1077158ef010b24cb5f9c, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:18:57,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741850_1026 (size=76) 2024-11-07T12:18:57,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:57,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing b472beaddbb1077158ef010b24cb5f9c, disabling compactions & flushes 2024-11-07T12:18:57,727 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:57,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:57,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. after waiting 0 ms 2024-11-07T12:18:57,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:57,727 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:57,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for b472beaddbb1077158ef010b24cb5f9c: 2024-11-07T12:18:57,729 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:18:57,729 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1730981937729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981937729"}]},"ts":"1730981937729"} 2024-11-07T12:18:57,731 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:18:57,732 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:18:57,732 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981937732"}]},"ts":"1730981937732"} 2024-11-07T12:18:57,734 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:18:57,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b472beaddbb1077158ef010b24cb5f9c, ASSIGN}] 2024-11-07T12:18:57,740 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b472beaddbb1077158ef010b24cb5f9c, ASSIGN 2024-11-07T12:18:57,741 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b472beaddbb1077158ef010b24cb5f9c, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:18:57,775 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:18:57,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:18:57,892 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=b472beaddbb1077158ef010b24cb5f9c, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:57,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure b472beaddbb1077158ef010b24cb5f9c, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:18:58,049 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:58,054 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:58,055 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => b472beaddbb1077158ef010b24cb5f9c, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:18:58,055 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,055 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:18:58,055 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,055 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,057 INFO [StoreOpener-b472beaddbb1077158ef010b24cb5f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,060 INFO [StoreOpener-b472beaddbb1077158ef010b24cb5f9c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b472beaddbb1077158ef010b24cb5f9c columnFamilyName testFamily 2024-11-07T12:18:58,060 DEBUG [StoreOpener-b472beaddbb1077158ef010b24cb5f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:18:58,060 INFO [StoreOpener-b472beaddbb1077158ef010b24cb5f9c-1 {}] regionserver.HStore(327): Store=b472beaddbb1077158ef010b24cb5f9c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:18:58,061 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,062 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,064 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:18:58,068 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:18:58,068 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened b472beaddbb1077158ef010b24cb5f9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62627736, jitterRate=-0.06677401065826416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:18:58,069 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for b472beaddbb1077158ef010b24cb5f9c: 2024-11-07T12:18:58,070 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c., pid=29, masterSystemTime=1730981938049 2024-11-07T12:18:58,072 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:58,072 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:18:58,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=b472beaddbb1077158ef010b24cb5f9c, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:18:58,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T12:18:58,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure b472beaddbb1077158ef010b24cb5f9c, server=fbad4c8a32e4,44527,1730981867254 in 179 msec 2024-11-07T12:18:58,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T12:18:58,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=b472beaddbb1077158ef010b24cb5f9c, ASSIGN in 339 msec 2024-11-07T12:18:58,080 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:18:58,080 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981938080"}]},"ts":"1730981938080"} 2024-11-07T12:18:58,082 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:18:58,085 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:18:58,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 787 msec 2024-11-07T12:18:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T12:18:58,305 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-07T12:18:58,306 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:18:58,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:58,310 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T12:18:58,310 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:18:58,310 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T12:18:58,312 DEBUG [hconnection-0x2caff318-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:58,314 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52164, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:58,325 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:18:58,328 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:18:58,328 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:18:58,330 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x29be1711 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:18:58,330 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x29be1711 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:18:58,332 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52178, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:18:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:18:58,336 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:18:58,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581492 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:58,337 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:18:58,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581492 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:18:58,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T12:19:01,342 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3005 2024-11-07T12:19:01,343 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62003296 to 127.0.0.1:51230 2024-11-07T12:19:01,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:01,343 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:01,343 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:01,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:01,343 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581492 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:01,343 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29be1711 to 127.0.0.1:51230 2024-11-07T12:19:01,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970124' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T12:19:01,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:52178 deadline: 1730981942343, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970124' does not exist 2024-11-07T12:19:01,346 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:19:01,346 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to fbad4c8a32e4,44527,1730981867254 for closing scanner id = -1672048081345970124 for b472beaddbb1077158ef010b24cb5f9c of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T12:19:01,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=227 (was 223) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 665), ProcessCount=11 (was 11), AvailableMemoryMB=1715 (was 1814) 2024-11-07T12:19:01,362 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=619, ProcessCount=11, AvailableMemoryMB=1714 2024-11-07T12:19:01,364 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x414fb75a to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f4c270a 2024-11-07T12:19:01,368 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x414fb75a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13578f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:01,369 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a1e7cdf to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ef86722 2024-11-07T12:19:01,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b1e2c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:01,376 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:01,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-07T12:19:01,379 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:01,379 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:01,379 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-07T12:19:01,380 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T12:19:01,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741851_1027 (size=417) 2024-11-07T12:19:01,793 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c2ca8fcef0a22d2abc17945041033cbb, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:01,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741852_1028 (size=68) 2024-11-07T12:19:01,814 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:01,814 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing c2ca8fcef0a22d2abc17945041033cbb, disabling compactions & flushes 2024-11-07T12:19:01,814 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:01,814 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:01,814 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. after waiting 0 ms 2024-11-07T12:19:01,814 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:01,814 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:01,814 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for c2ca8fcef0a22d2abc17945041033cbb: 2024-11-07T12:19:01,816 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:01,817 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1730981941816"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981941816"}]},"ts":"1730981941816"} 2024-11-07T12:19:01,820 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:01,821 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:01,821 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981941821"}]},"ts":"1730981941821"} 2024-11-07T12:19:01,824 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:01,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=c2ca8fcef0a22d2abc17945041033cbb, ASSIGN}] 2024-11-07T12:19:01,830 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=c2ca8fcef0a22d2abc17945041033cbb, ASSIGN 2024-11-07T12:19:01,831 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=c2ca8fcef0a22d2abc17945041033cbb, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:01,838 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:19:01,982 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=c2ca8fcef0a22d2abc17945041033cbb, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:01,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure c2ca8fcef0a22d2abc17945041033cbb, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:02,140 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:02,147 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:02,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => c2ca8fcef0a22d2abc17945041033cbb, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:02,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:02,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,151 INFO [StoreOpener-c2ca8fcef0a22d2abc17945041033cbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,153 INFO [StoreOpener-c2ca8fcef0a22d2abc17945041033cbb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2ca8fcef0a22d2abc17945041033cbb columnFamilyName testFamily 2024-11-07T12:19:02,153 DEBUG [StoreOpener-c2ca8fcef0a22d2abc17945041033cbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:02,154 INFO [StoreOpener-c2ca8fcef0a22d2abc17945041033cbb-1 {}] regionserver.HStore(327): Store=c2ca8fcef0a22d2abc17945041033cbb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:02,156 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,157 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,160 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:19:02,164 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:02,165 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened c2ca8fcef0a22d2abc17945041033cbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75007076, jitterRate=0.11769253015518188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:02,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for c2ca8fcef0a22d2abc17945041033cbb: 2024-11-07T12:19:02,168 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb., pid=32, masterSystemTime=1730981942140 2024-11-07T12:19:02,170 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:02,170 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:19:02,171 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=c2ca8fcef0a22d2abc17945041033cbb, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:02,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T12:19:02,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure c2ca8fcef0a22d2abc17945041033cbb, server=fbad4c8a32e4,44527,1730981867254 in 188 msec 2024-11-07T12:19:02,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T12:19:02,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=c2ca8fcef0a22d2abc17945041033cbb, ASSIGN in 347 msec 2024-11-07T12:19:02,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:02,179 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981942179"}]},"ts":"1730981942179"} 2024-11-07T12:19:02,181 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:02,184 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:02,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 808 msec 2024-11-07T12:19:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T12:19:02,383 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-07T12:19:02,384 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:02,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:02,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:02,389 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:02,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-07T12:19:02,392 DEBUG [hconnection-0x55c76ff-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:02,394 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38936, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:02,405 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:02,408 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:19:02,411 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581496 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:02,412 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T12:19:02,805 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-07T12:19:02,806 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-07T12:19:03,191 INFO [master/fbad4c8a32e4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T12:19:03,191 INFO [master/fbad4c8a32e4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T12:19:03,557 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:06,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T12:19:06,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:06,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-07T12:19:06,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:08,913 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6503 2024-11-07T12:19:08,913 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a1e7cdf to 127.0.0.1:51230 2024-11-07T12:19:08,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:08,913 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:08,913 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:08,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:08,913 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x414fb75a to 127.0.0.1:51230 2024-11-07T12:19:08,928 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227 (was 227), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=578 (was 619), ProcessCount=11 (was 11), AvailableMemoryMB=1629 (was 1714) 2024-11-07T12:19:08,941 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=578, ProcessCount=11, AvailableMemoryMB=1629 2024-11-07T12:19:08,944 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16a36c45 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6fe86375 2024-11-07T12:19:08,951 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x16a36c45 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d4b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:08,953 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cd84775 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@246233e2 2024-11-07T12:19:08,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292ea26f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:08,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-07T12:19:08,976 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:08,976 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:08,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-07T12:19:08,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T12:19:09,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741853_1029 (size=421) 2024-11-07T12:19:09,414 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 46a4386907010562876d51566682636a, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:09,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741854_1030 (size=72) 2024-11-07T12:19:09,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:09,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 46a4386907010562876d51566682636a, disabling compactions & flushes 2024-11-07T12:19:09,463 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. after waiting 0 ms 2024-11-07T12:19:09,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,463 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,464 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 46a4386907010562876d51566682636a: 2024-11-07T12:19:09,467 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:09,467 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730981949467"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981949467"}]},"ts":"1730981949467"} 2024-11-07T12:19:09,472 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:09,475 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:09,475 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981949475"}]},"ts":"1730981949475"} 2024-11-07T12:19:09,478 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:09,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=46a4386907010562876d51566682636a, ASSIGN}] 2024-11-07T12:19:09,485 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=46a4386907010562876d51566682636a, ASSIGN 2024-11-07T12:19:09,487 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=46a4386907010562876d51566682636a, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:09,638 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=46a4386907010562876d51566682636a, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:09,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 46a4386907010562876d51566682636a, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:09,796 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:09,805 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,805 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => 46a4386907010562876d51566682636a, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:09,805 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 46a4386907010562876d51566682636a 2024-11-07T12:19:09,806 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:09,806 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for 46a4386907010562876d51566682636a 2024-11-07T12:19:09,806 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for 46a4386907010562876d51566682636a 2024-11-07T12:19:09,808 INFO [StoreOpener-46a4386907010562876d51566682636a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 46a4386907010562876d51566682636a 2024-11-07T12:19:09,812 INFO [StoreOpener-46a4386907010562876d51566682636a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46a4386907010562876d51566682636a columnFamilyName testFamily 2024-11-07T12:19:09,812 DEBUG [StoreOpener-46a4386907010562876d51566682636a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:09,813 INFO [StoreOpener-46a4386907010562876d51566682636a-1 {}] regionserver.HStore(327): Store=46a4386907010562876d51566682636a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:09,814 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a 2024-11-07T12:19:09,815 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a 2024-11-07T12:19:09,818 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for 46a4386907010562876d51566682636a 2024-11-07T12:19:09,824 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:09,825 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened 46a4386907010562876d51566682636a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71203442, jitterRate=0.06101396679878235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:09,826 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for 46a4386907010562876d51566682636a: 2024-11-07T12:19:09,828 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a., pid=35, masterSystemTime=1730981949796 2024-11-07T12:19:09,832 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,832 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:19:09,833 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=46a4386907010562876d51566682636a, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:09,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T12:19:09,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 46a4386907010562876d51566682636a, server=fbad4c8a32e4,44527,1730981867254 in 194 msec 2024-11-07T12:19:09,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T12:19:09,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=46a4386907010562876d51566682636a, ASSIGN in 357 msec 2024-11-07T12:19:09,845 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:09,845 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981949845"}]},"ts":"1730981949845"} 2024-11-07T12:19:09,848 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:09,852 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:09,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 879 msec 2024-11-07T12:19:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T12:19:09,984 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-07T12:19:09,984 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:09,984 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:10,005 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:10,006 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:10,006 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-07T12:19:10,015 DEBUG [hconnection-0x2275ac1d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:10,018 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46540, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:10,032 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:10,037 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:19:10,043 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581500 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:10,043 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T12:19:11,544 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1507 2024-11-07T12:19:11,545 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0cd84775 to 127.0.0.1:51230 2024-11-07T12:19:11,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:11,545 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:11,545 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:11,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:11,545 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x16a36c45 to 127.0.0.1:51230 2024-11-07T12:19:11,559 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=230 (was 227) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=595 (was 578) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1540 (was 1629) 2024-11-07T12:19:11,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=230, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=595, ProcessCount=11, AvailableMemoryMB=1540 2024-11-07T12:19:11,572 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x508551b0 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@63aba1c0 2024-11-07T12:19:11,576 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x508551b0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192c8a5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:11,578 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29caf2bf to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58063b0a 2024-11-07T12:19:11,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f4f96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:11,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-07T12:19:11,603 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:11,603 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:11,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-07T12:19:11,605 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T12:19:11,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741855_1031 (size=420) 2024-11-07T12:19:12,076 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 57fa8c5121af12042d7adaedb84ee993, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:12,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741856_1032 (size=71) 2024-11-07T12:19:12,114 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:12,115 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 57fa8c5121af12042d7adaedb84ee993, disabling compactions & flushes 2024-11-07T12:19:12,115 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,115 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,115 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. after waiting 0 ms 2024-11-07T12:19:12,115 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,115 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,115 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 57fa8c5121af12042d7adaedb84ee993: 2024-11-07T12:19:12,116 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:12,117 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730981952116"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981952116"}]},"ts":"1730981952116"} 2024-11-07T12:19:12,120 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:12,123 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:12,124 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981952123"}]},"ts":"1730981952123"} 2024-11-07T12:19:12,127 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:12,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=57fa8c5121af12042d7adaedb84ee993, ASSIGN}] 2024-11-07T12:19:12,137 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=57fa8c5121af12042d7adaedb84ee993, ASSIGN 2024-11-07T12:19:12,139 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=57fa8c5121af12042d7adaedb84ee993, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:12,290 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=57fa8c5121af12042d7adaedb84ee993, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:12,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 57fa8c5121af12042d7adaedb84ee993, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:12,453 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:12,474 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,474 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 57fa8c5121af12042d7adaedb84ee993, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:12,475 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,475 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:12,475 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,475 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,479 INFO [StoreOpener-57fa8c5121af12042d7adaedb84ee993-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,482 INFO [StoreOpener-57fa8c5121af12042d7adaedb84ee993-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57fa8c5121af12042d7adaedb84ee993 columnFamilyName testFamily 2024-11-07T12:19:12,483 DEBUG [StoreOpener-57fa8c5121af12042d7adaedb84ee993-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:12,486 INFO [StoreOpener-57fa8c5121af12042d7adaedb84ee993-1 {}] regionserver.HStore(327): Store=57fa8c5121af12042d7adaedb84ee993/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:12,487 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,497 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,501 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:19:12,514 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:12,516 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 57fa8c5121af12042d7adaedb84ee993; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67724706, jitterRate=0.009176760911941528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:12,520 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 57fa8c5121af12042d7adaedb84ee993: 2024-11-07T12:19:12,522 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993., pid=38, masterSystemTime=1730981952453 2024-11-07T12:19:12,525 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,525 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:19:12,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=57fa8c5121af12042d7adaedb84ee993, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:12,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T12:19:12,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 57fa8c5121af12042d7adaedb84ee993, server=fbad4c8a32e4,44527,1730981867254 in 231 msec 2024-11-07T12:19:12,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T12:19:12,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=57fa8c5121af12042d7adaedb84ee993, ASSIGN in 395 msec 2024-11-07T12:19:12,535 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:12,535 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981952535"}]},"ts":"1730981952535"} 2024-11-07T12:19:12,538 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:12,543 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:12,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 945 msec 2024-11-07T12:19:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T12:19:12,609 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-07T12:19:12,609 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:12,610 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:12,624 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:12,624 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:12,624 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-07T12:19:12,645 DEBUG [hconnection-0xec399e1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:12,656 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46556, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:12,674 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:12,698 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b2b8664 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69ce4d69 2024-11-07T12:19:12,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d01f0cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:12,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:19:12,728 DEBUG [hconnection-0x4fbe7b7b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:12,734 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46568, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:12,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:19:12,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581505 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:12,755 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:19:12,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581505 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:12,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T12:19:13,763 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T12:19:13.762Z, RpcRetryingCaller{globalStartTime=2024-11-07T12:19:12.755Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T12:19:13.762Z, RpcRetryingCaller{globalStartTime=2024-11-07T12:19:12.755Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:19:13,763 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1008 2024-11-07T12:19:13,764 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b2b8664 to 127.0.0.1:51230 2024-11-07T12:19:13,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:13,765 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29caf2bf to 127.0.0.1:51230 2024-11-07T12:19:13,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:13,766 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:13,766 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:13,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:13,766 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x508551b0 to 127.0.0.1:51230 2024-11-07T12:19:13,781 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=235 (was 230) - Thread LEAK? -, OpenFileDescriptor=459 (was 456) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=595 (was 595), ProcessCount=11 (was 11), AvailableMemoryMB=1494 (was 1540) 2024-11-07T12:19:13,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=235, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=595, ProcessCount=11, AvailableMemoryMB=1494 2024-11-07T12:19:13,806 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13353321 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@222723ad 2024-11-07T12:19:13,827 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x13353321 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a553171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:13,830 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dec9aae to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a11225f 2024-11-07T12:19:13,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5007b807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:13,867 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:13,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T12:19:13,871 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:13,871 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:13,871 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-07T12:19:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T12:19:13,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:13,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741857_1033 (size=422) 2024-11-07T12:19:14,016 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:19:14,257 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:19:14,301 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 15982a8bd19de3fe41d3798072bfe2a3, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:14,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741858_1034 (size=73) 2024-11-07T12:19:14,713 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:14,713 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 15982a8bd19de3fe41d3798072bfe2a3, disabling compactions & flushes 2024-11-07T12:19:14,714 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:14,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:14,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. after waiting 0 ms 2024-11-07T12:19:14,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:14,714 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:14,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 15982a8bd19de3fe41d3798072bfe2a3: 2024-11-07T12:19:14,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:14,716 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1730981954716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981954716"}]},"ts":"1730981954716"} 2024-11-07T12:19:14,722 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:14,723 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:14,724 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981954723"}]},"ts":"1730981954723"} 2024-11-07T12:19:14,729 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:14,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=15982a8bd19de3fe41d3798072bfe2a3, ASSIGN}] 2024-11-07T12:19:14,736 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=15982a8bd19de3fe41d3798072bfe2a3, ASSIGN 2024-11-07T12:19:14,738 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=15982a8bd19de3fe41d3798072bfe2a3, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T12:19:14,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=15982a8bd19de3fe41d3798072bfe2a3, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:14,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 15982a8bd19de3fe41d3798072bfe2a3, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:15,050 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:15,060 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:15,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 15982a8bd19de3fe41d3798072bfe2a3, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:15,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:15,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,069 INFO [StoreOpener-15982a8bd19de3fe41d3798072bfe2a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,073 INFO [StoreOpener-15982a8bd19de3fe41d3798072bfe2a3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15982a8bd19de3fe41d3798072bfe2a3 columnFamilyName testFamily 2024-11-07T12:19:15,073 DEBUG [StoreOpener-15982a8bd19de3fe41d3798072bfe2a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:15,074 INFO [StoreOpener-15982a8bd19de3fe41d3798072bfe2a3-1 {}] regionserver.HStore(327): Store=15982a8bd19de3fe41d3798072bfe2a3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:15,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,077 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:19:15,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:15,097 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 15982a8bd19de3fe41d3798072bfe2a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66915485, jitterRate=-0.002881571650505066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:15,098 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 15982a8bd19de3fe41d3798072bfe2a3: 2024-11-07T12:19:15,101 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3., pid=41, masterSystemTime=1730981955049 2024-11-07T12:19:15,104 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:15,104 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:19:15,105 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=15982a8bd19de3fe41d3798072bfe2a3, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:15,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T12:19:15,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 15982a8bd19de3fe41d3798072bfe2a3, server=fbad4c8a32e4,44527,1730981867254 in 216 msec 2024-11-07T12:19:15,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T12:19:15,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=15982a8bd19de3fe41d3798072bfe2a3, ASSIGN in 377 msec 2024-11-07T12:19:15,117 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:15,117 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981955117"}]},"ts":"1730981955117"} 2024-11-07T12:19:15,119 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:15,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:15,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.2570 sec 2024-11-07T12:19:15,355 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:15,396 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-07T12:19:15,397 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-07T12:19:15,398 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-07T12:19:16,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-07T12:19:16,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:16,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T12:19:16,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:16,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T12:19:16,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T12:19:16,887 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-07T12:19:16,887 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:16,887 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:16,895 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:16,895 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:16,895 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T12:19:16,901 DEBUG [hconnection-0x3c2c9bbf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:16,902 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:16,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:16,929 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:19:16,941 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:19:16,941 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T12:19:20,563 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:22,208 DEBUG [master/fbad4c8a32e4:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T12:19:22,213 DEBUG [master/fbad4c8a32e4:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 69f29002b8ac38e61dd30e67f2cf97a7 changed from -1.0 to 0.0, refreshing cache 2024-11-07T12:19:22,414 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:33401,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:19:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T12:19:23,980 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:19:23,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:46572 deadline: 1730981965942, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:19:23,983 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:19:23,983 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581509 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:19:23,984 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7054 2024-11-07T12:19:23,985 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1dec9aae to 127.0.0.1:51230 2024-11-07T12:19:23,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:23,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:23,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:23,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:23,985 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13353321 to 127.0.0.1:51230 2024-11-07T12:19:24,004 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=229 (was 235), OpenFileDescriptor=450 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=651 (was 595) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2140 (was 1494) - AvailableMemoryMB LEAK? - 2024-11-07T12:19:24,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=651, ProcessCount=11, AvailableMemoryMB=2138 2024-11-07T12:19:24,037 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55ff9601 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10e2a98c 2024-11-07T12:19:24,058 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x55ff9601 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8330c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:24,060 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59f6fab7 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a366dc 2024-11-07T12:19:24,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b45b717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:24,096 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:24,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T12:19:24,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:24,099 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:24,100 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-07T12:19:24,101 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T12:19:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741859_1035 (size=428) 2024-11-07T12:19:24,183 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1545cc022e3b8dd3b28b3dd70139e99f, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:24,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741860_1036 (size=79) 2024-11-07T12:19:24,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:24,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 1545cc022e3b8dd3b28b3dd70139e99f, disabling compactions & flushes 2024-11-07T12:19:24,266 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,266 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,266 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. after waiting 0 ms 2024-11-07T12:19:24,266 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,266 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,266 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1545cc022e3b8dd3b28b3dd70139e99f: 2024-11-07T12:19:24,268 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:24,269 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1730981964269"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981964269"}]},"ts":"1730981964269"} 2024-11-07T12:19:24,274 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:24,275 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:24,275 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981964275"}]},"ts":"1730981964275"} 2024-11-07T12:19:24,279 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:24,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1545cc022e3b8dd3b28b3dd70139e99f, ASSIGN}] 2024-11-07T12:19:24,302 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1545cc022e3b8dd3b28b3dd70139e99f, ASSIGN 2024-11-07T12:19:24,304 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1545cc022e3b8dd3b28b3dd70139e99f, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:24,459 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=1545cc022e3b8dd3b28b3dd70139e99f, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:24,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 1545cc022e3b8dd3b28b3dd70139e99f, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:24,619 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:24,644 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 1545cc022e3b8dd3b28b3dd70139e99f, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:24,652 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,652 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:24,652 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,652 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,668 INFO [StoreOpener-1545cc022e3b8dd3b28b3dd70139e99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,682 INFO [StoreOpener-1545cc022e3b8dd3b28b3dd70139e99f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1545cc022e3b8dd3b28b3dd70139e99f columnFamilyName testFamily 2024-11-07T12:19:24,682 DEBUG [StoreOpener-1545cc022e3b8dd3b28b3dd70139e99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:24,685 INFO [StoreOpener-1545cc022e3b8dd3b28b3dd70139e99f-1 {}] regionserver.HStore(327): Store=1545cc022e3b8dd3b28b3dd70139e99f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:24,689 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,701 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:19:24,727 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:24,728 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 1545cc022e3b8dd3b28b3dd70139e99f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62846133, jitterRate=-0.06351964175701141}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:24,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 1545cc022e3b8dd3b28b3dd70139e99f: 2024-11-07T12:19:24,731 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., pid=44, masterSystemTime=1730981964619 2024-11-07T12:19:24,735 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,735 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:19:24,735 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=1545cc022e3b8dd3b28b3dd70139e99f, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:24,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-07T12:19:24,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 1545cc022e3b8dd3b28b3dd70139e99f, server=fbad4c8a32e4,44527,1730981867254 in 275 msec 2024-11-07T12:19:24,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T12:19:24,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1545cc022e3b8dd3b28b3dd70139e99f, ASSIGN in 443 msec 2024-11-07T12:19:24,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:24,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981964745"}]},"ts":"1730981964745"} 2024-11-07T12:19:24,748 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:24,757 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:24,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 662 msec 2024-11-07T12:19:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T12:19:25,108 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-07T12:19:25,108 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:25,108 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:25,120 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:25,120 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:25,120 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T12:19:25,141 DEBUG [hconnection-0x5a3d107e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:25,143 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54554, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:25,171 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:25,201 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:19:25,201 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:19:25,220 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x55ff9601 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:19:25,221 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x55ff9601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:25,222 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54562, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:19:25,236 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:19:26,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T12:19:26,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:27,316 ERROR [LeaseRenewer:jenkins@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:33401,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:19:27,400 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T12:19:30,154 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:31,233 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-07T12:19:31,234 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-07T12:19:31,234 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:19:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T12:19:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:19:32,252 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:19:34,202 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1950ms, rpcTimeout=1949ms 2024-11-07T12:19:34,202 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1950ms, rpcTimeout=1949ms 2024-11-07T12:19:34,202 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:19:34,203 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T12:19:31.234Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-07T12:19:34.202Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1950ms, rpcTimeout=1949ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1950ms, rpcTimeout=1949ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1950ms, rpcTimeout=1949ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T12:19:34,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9004 2024-11-07T12:19:34,205 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59f6fab7 to 127.0.0.1:51230 2024-11-07T12:19:34,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:34,206 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:34,206 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:34,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:34,206 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55ff9601 to 127.0.0.1:51230 2024-11-07T12:19:34,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229 (was 229), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=669 (was 651) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1450 (was 2138) 2024-11-07T12:19:34,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=669, ProcessCount=11, AvailableMemoryMB=1450 2024-11-07T12:19:34,238 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0ef34ecf to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e128fcc 2024-11-07T12:19:34,243 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x0ef34ecf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69bf5f65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:34,245 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f666971 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f186bb5 2024-11-07T12:19:34,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b09cdaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:34,259 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:34,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-07T12:19:34,262 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:34,263 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:34,263 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-07T12:19:34,264 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T12:19:34,270 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970156 expired clientIPAndPort=172.17.0.2:38390, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:19:34,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741861_1037 (size=416) 2024-11-07T12:19:34,686 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 40f79a6cfe590363e2b8a7ecd6208f96, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:34,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741862_1038 (size=67) 2024-11-07T12:19:35,112 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:35,112 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 40f79a6cfe590363e2b8a7ecd6208f96, disabling compactions & flushes 2024-11-07T12:19:35,112 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,112 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,112 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. after waiting 0 ms 2024-11-07T12:19:35,112 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,112 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,112 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 40f79a6cfe590363e2b8a7ecd6208f96: 2024-11-07T12:19:35,114 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:35,115 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1730981975114"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981975114"}]},"ts":"1730981975114"} 2024-11-07T12:19:35,117 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:35,118 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:35,119 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981975119"}]},"ts":"1730981975119"} 2024-11-07T12:19:35,121 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:35,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=40f79a6cfe590363e2b8a7ecd6208f96, ASSIGN}] 2024-11-07T12:19:35,132 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=40f79a6cfe590363e2b8a7ecd6208f96, ASSIGN 2024-11-07T12:19:35,133 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=40f79a6cfe590363e2b8a7ecd6208f96, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T12:19:35,284 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=40f79a6cfe590363e2b8a7ecd6208f96, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:35,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 40f79a6cfe590363e2b8a7ecd6208f96, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:35,443 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:35,451 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 40f79a6cfe590363e2b8a7ecd6208f96, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:35,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:35,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,456 INFO [StoreOpener-40f79a6cfe590363e2b8a7ecd6208f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,458 INFO [StoreOpener-40f79a6cfe590363e2b8a7ecd6208f96-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f79a6cfe590363e2b8a7ecd6208f96 columnFamilyName testFamily 2024-11-07T12:19:35,459 DEBUG [StoreOpener-40f79a6cfe590363e2b8a7ecd6208f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:35,459 INFO [StoreOpener-40f79a6cfe590363e2b8a7ecd6208f96-1 {}] regionserver.HStore(327): Store=40f79a6cfe590363e2b8a7ecd6208f96/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:35,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,461 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,468 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:19:35,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:35,473 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 40f79a6cfe590363e2b8a7ecd6208f96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66398404, jitterRate=-0.010586678981781006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:35,474 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 40f79a6cfe590363e2b8a7ecd6208f96: 2024-11-07T12:19:35,475 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96., pid=47, masterSystemTime=1730981975443 2024-11-07T12:19:35,477 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,477 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:19:35,478 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=40f79a6cfe590363e2b8a7ecd6208f96, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:35,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T12:19:35,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 40f79a6cfe590363e2b8a7ecd6208f96, server=fbad4c8a32e4,44527,1730981867254 in 191 msec 2024-11-07T12:19:35,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-07T12:19:35,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=40f79a6cfe590363e2b8a7ecd6208f96, ASSIGN in 352 msec 2024-11-07T12:19:35,486 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:35,486 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981975486"}]},"ts":"1730981975486"} 2024-11-07T12:19:35,488 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:35,492 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:35,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.2320 sec 2024-11-07T12:19:36,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-07T12:19:36,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T12:19:37,270 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-07T12:19:37,270 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:37,270 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:37,275 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:37,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:37,276 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-07T12:19:37,279 DEBUG [hconnection-0x64b6c7a7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:37,280 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54002, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:37,307 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:37,312 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48d9bd50 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64828ea0 2024-11-07T12:19:37,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31343507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:37,320 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:19:37,345 DEBUG [hconnection-0x41b34a0f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:37,352 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54004, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:37,372 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:19:37,377 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581522 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:37,378 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:19:37,379 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581522 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:37,380 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T12:19:38,753 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:19:39,402 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-07T12:19:41,003 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:41,285 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970154 expired clientIPAndPort=172.17.0.2:38390, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:19:43,382 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T12:19:43.382Z, RpcRetryingCaller{globalStartTime=2024-11-07T12:19:37.379Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T12:19:43.382Z, RpcRetryingCaller{globalStartTime=2024-11-07T12:19:37.379Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:19:43,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6004 2024-11-07T12:19:43,383 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x48d9bd50 to 127.0.0.1:51230 2024-11-07T12:19:43,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:43,383 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f666971 to 127.0.0.1:51230 2024-11-07T12:19:43,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:43,383 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:43,383 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:43,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:43,384 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0ef34ecf to 127.0.0.1:51230 2024-11-07T12:19:43,395 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=233 (was 229) - Thread LEAK? -, OpenFileDescriptor=451 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=671 (was 669) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1196 (was 1450) 2024-11-07T12:19:43,407 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=233, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=671, ProcessCount=11, AvailableMemoryMB=1195 2024-11-07T12:19:43,410 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x44b92d34 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a0efe6f 2024-11-07T12:19:43,417 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x44b92d34 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b39ed4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:43,418 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x158538c4 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55fa340a 2024-11-07T12:19:43,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5827630c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:43,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:43,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T12:19:43,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:43,429 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:43,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-07T12:19:43,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T12:19:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741863_1039 (size=432) 2024-11-07T12:19:43,849 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4ceec90215644686ca88a4a9060a8a10, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:43,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741864_1040 (size=83) 2024-11-07T12:19:43,880 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:19:44,017 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:19:44,264 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:44,264 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 4ceec90215644686ca88a4a9060a8a10, disabling compactions & flushes 2024-11-07T12:19:44,264 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,264 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,264 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. after waiting 0 ms 2024-11-07T12:19:44,264 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,264 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,264 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4ceec90215644686ca88a4a9060a8a10: 2024-11-07T12:19:44,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:44,266 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1730981984266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981984266"}]},"ts":"1730981984266"} 2024-11-07T12:19:44,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:44,274 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:44,274 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981984274"}]},"ts":"1730981984274"} 2024-11-07T12:19:44,277 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T12:19:44,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=4ceec90215644686ca88a4a9060a8a10, ASSIGN}] 2024-11-07T12:19:44,284 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=4ceec90215644686ca88a4a9060a8a10, ASSIGN 2024-11-07T12:19:44,285 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=4ceec90215644686ca88a4a9060a8a10, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T12:19:44,436 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=4ceec90215644686ca88a4a9060a8a10, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:44,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 4ceec90215644686ca88a4a9060a8a10, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:44,590 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:44,597 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,597 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => 4ceec90215644686ca88a4a9060a8a10, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:44,598 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,598 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:44,598 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for 4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,598 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for 4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,606 INFO [StoreOpener-4ceec90215644686ca88a4a9060a8a10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,609 INFO [StoreOpener-4ceec90215644686ca88a4a9060a8a10-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ceec90215644686ca88a4a9060a8a10 columnFamilyName testFamily 2024-11-07T12:19:44,609 DEBUG [StoreOpener-4ceec90215644686ca88a4a9060a8a10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:44,610 INFO [StoreOpener-4ceec90215644686ca88a4a9060a8a10-1 {}] regionserver.HStore(327): Store=4ceec90215644686ca88a4a9060a8a10/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:44,611 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,612 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,617 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for 4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:19:44,625 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:44,626 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened 4ceec90215644686ca88a4a9060a8a10; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61644166, jitterRate=-0.0814303457736969}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:44,627 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for 4ceec90215644686ca88a4a9060a8a10: 2024-11-07T12:19:44,628 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., pid=50, masterSystemTime=1730981984590 2024-11-07T12:19:44,631 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,631 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:19:44,631 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=4ceec90215644686ca88a4a9060a8a10, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:44,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-07T12:19:44,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 4ceec90215644686ca88a4a9060a8a10, server=fbad4c8a32e4,44527,1730981867254 in 196 msec 2024-11-07T12:19:44,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T12:19:44,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=4ceec90215644686ca88a4a9060a8a10, ASSIGN in 355 msec 2024-11-07T12:19:44,641 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:44,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981984641"}]},"ts":"1730981984641"} 2024-11-07T12:19:44,644 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T12:19:44,647 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:44,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.2200 sec 2024-11-07T12:19:45,403 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T12:19:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T12:19:46,437 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-07T12:19:46,437 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T12:19:46,437 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:46,443 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T12:19:46,444 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:46,444 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T12:19:46,448 DEBUG [hconnection-0xe2a406d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:46,451 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33808, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:46,463 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:46,468 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:19:46,468 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:19:46,470 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x44b92d34 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:19:46,470 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x44b92d34 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:46,472 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33816, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:19:46,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:19:46,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T12:19:46,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:47,295 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970149 expired clientIPAndPort=172.17.0.2:54510, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:19:47,483 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-07T12:19:47,483 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-07T12:19:47,483 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:19:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T12:19:47,996 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970145 expired clientIPAndPort=172.17.0.2:50166, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:19:47,996 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970143 expired clientIPAndPort=172.17.0.2:50166, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:19:47,996 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970142 expired clientIPAndPort=172.17.0.2:50166, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:19:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:19:48,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:19:49,472 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms 2024-11-07T12:19:49,473 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms 2024-11-07T12:19:49,473 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:19:49,473 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T12:19:47.483Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-07T12:19:49.473Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T12:19:49,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3009 2024-11-07T12:19:49,478 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x158538c4 to 127.0.0.1:51230 2024-11-07T12:19:49,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:49,478 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:49,478 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:49,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:49,478 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x44b92d34 to 127.0.0.1:51230 2024-11-07T12:19:49,494 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=229 (was 233), OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 671) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=889 (was 1195) 2024-11-07T12:19:49,507 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=888 2024-11-07T12:19:49,511 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x328fc448 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@660292f0 2024-11-07T12:19:49,515 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x328fc448 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4191e7f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:49,517 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01467966 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4dddd3a5 2024-11-07T12:19:49,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bcf3d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:49,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T12:19:49,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:49,531 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:49,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-07T12:19:49,532 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T12:19:49,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741865_1041 (size=436) 2024-11-07T12:19:49,563 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6afeba0ade51e5d04366bab1ccc89b28, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741866_1042 (size=87) 2024-11-07T12:19:49,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:49,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 6afeba0ade51e5d04366bab1ccc89b28, disabling compactions & flushes 2024-11-07T12:19:49,979 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:49,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:49,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. after waiting 0 ms 2024-11-07T12:19:49,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:49,979 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:49,979 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6afeba0ade51e5d04366bab1ccc89b28: 2024-11-07T12:19:49,981 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:49,981 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1730981989981"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981989981"}]},"ts":"1730981989981"} 2024-11-07T12:19:49,984 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:49,985 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:49,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981989985"}]},"ts":"1730981989985"} 2024-11-07T12:19:49,987 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:19:50,001 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:19:50,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6afeba0ade51e5d04366bab1ccc89b28, ASSIGN}] 2024-11-07T12:19:50,006 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6afeba0ade51e5d04366bab1ccc89b28, ASSIGN 2024-11-07T12:19:50,008 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6afeba0ade51e5d04366bab1ccc89b28, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:50,100 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:50,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=6afeba0ade51e5d04366bab1ccc89b28, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:50,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 6afeba0ade51e5d04366bab1ccc89b28, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:50,315 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:50,321 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:50,322 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => 6afeba0ade51e5d04366bab1ccc89b28, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:50,322 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,322 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:50,322 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for 6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,322 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for 6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,333 INFO [StoreOpener-6afeba0ade51e5d04366bab1ccc89b28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,336 INFO [StoreOpener-6afeba0ade51e5d04366bab1ccc89b28-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6afeba0ade51e5d04366bab1ccc89b28 columnFamilyName testFamily 2024-11-07T12:19:50,337 DEBUG [StoreOpener-6afeba0ade51e5d04366bab1ccc89b28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:50,340 INFO [StoreOpener-6afeba0ade51e5d04366bab1ccc89b28-1 {}] regionserver.HStore(327): Store=6afeba0ade51e5d04366bab1ccc89b28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:50,342 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,343 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,347 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for 6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:19:50,351 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:50,352 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened 6afeba0ade51e5d04366bab1ccc89b28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69115544, jitterRate=0.029901862144470215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:50,355 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for 6afeba0ade51e5d04366bab1ccc89b28: 2024-11-07T12:19:50,357 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28., pid=53, masterSystemTime=1730981990315 2024-11-07T12:19:50,361 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:50,361 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:19:50,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=6afeba0ade51e5d04366bab1ccc89b28, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:50,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T12:19:50,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 6afeba0ade51e5d04366bab1ccc89b28, server=fbad4c8a32e4,44527,1730981867254 in 202 msec 2024-11-07T12:19:50,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-07T12:19:50,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=6afeba0ade51e5d04366bab1ccc89b28, ASSIGN in 363 msec 2024-11-07T12:19:50,371 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:50,372 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981990371"}]},"ts":"1730981990371"} 2024-11-07T12:19:50,375 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:19:50,380 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:50,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 852 msec 2024-11-07T12:19:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T12:19:50,536 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-07T12:19:50,536 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:19:50,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:50,544 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T12:19:50,545 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:50,545 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-07T12:19:50,547 DEBUG [hconnection-0x4c350d0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:50,549 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47338, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:50,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:50,585 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T12:19:50,585 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:19:50,587 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x328fc448 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:19:50,587 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x328fc448 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:50,589 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47348, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:19:50,594 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T12:19:50,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581534 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:50,596 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T12:19:50,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581534 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:50,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970082 2024-11-07T12:19:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:47348 deadline: 1730981993596, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:19:50,599 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581534 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:50,600 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T12:19:50,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581535 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:50,601 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T12:19:50,602 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:19:50,603 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:19:50,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581536 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:50,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970080 2024-11-07T12:19:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:47348 deadline: 1730981993602, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:19:50,607 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:19:50,607 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581536 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:50,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581537 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:50,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1672048081345970079 2024-11-07T12:19:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:47348 deadline: 1730981993607, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:19:50,610 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581537 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:50,611 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:19:50,612 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581538 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:19:50,612 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1672048081345970078 2024-11-07T12:19:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:47348 deadline: 1730981993610, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:19:50,613 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581538 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:19:50,615 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:19:50,615 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01467966 to 127.0.0.1:51230 2024-11-07T12:19:50,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:50,616 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:19:50,616 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:19:50,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:19:50,616 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x328fc448 to 127.0.0.1:51230 2024-11-07T12:19:50,635 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232 (was 229) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=682 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=857 (was 888) 2024-11-07T12:19:50,653 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=682, ProcessCount=11, AvailableMemoryMB=857 2024-11-07T12:19:50,658 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32e97093 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7012e8b4 2024-11-07T12:19:50,689 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x32e97093 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c2ae93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:50,691 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e81c4b4 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@377ae836 2024-11-07T12:19:50,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eee6ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:19:50,715 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:19:50,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T12:19:50,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:19:50,718 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:50,720 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-07T12:19:50,720 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:19:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T12:19:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741867_1043 (size=422) 2024-11-07T12:19:50,777 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bebca667db35bff8ba945d067d8e9101, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:19:50,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741868_1044 (size=73) 2024-11-07T12:19:50,822 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:50,822 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing bebca667db35bff8ba945d067d8e9101, disabling compactions & flushes 2024-11-07T12:19:50,822 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:50,822 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:50,822 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. after waiting 0 ms 2024-11-07T12:19:50,822 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:50,822 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:50,822 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for bebca667db35bff8ba945d067d8e9101: 2024-11-07T12:19:50,825 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:19:50,825 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1730981990825"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730981990825"}]},"ts":"1730981990825"} 2024-11-07T12:19:50,828 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:19:50,832 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:19:50,832 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981990832"}]},"ts":"1730981990832"} 2024-11-07T12:19:50,835 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:19:50,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=bebca667db35bff8ba945d067d8e9101, ASSIGN}] 2024-11-07T12:19:50,842 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=bebca667db35bff8ba945d067d8e9101, ASSIGN 2024-11-07T12:19:50,842 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=bebca667db35bff8ba945d067d8e9101, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:19:50,993 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=bebca667db35bff8ba945d067d8e9101, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:50,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure bebca667db35bff8ba945d067d8e9101, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:19:51,148 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:51,155 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:51,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => bebca667db35bff8ba945d067d8e9101, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:19:51,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,156 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:19:51,156 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,156 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,162 INFO [StoreOpener-bebca667db35bff8ba945d067d8e9101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,170 INFO [StoreOpener-bebca667db35bff8ba945d067d8e9101-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bebca667db35bff8ba945d067d8e9101 columnFamilyName testFamily 2024-11-07T12:19:51,170 DEBUG [StoreOpener-bebca667db35bff8ba945d067d8e9101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:19:51,171 INFO [StoreOpener-bebca667db35bff8ba945d067d8e9101-1 {}] regionserver.HStore(327): Store=bebca667db35bff8ba945d067d8e9101/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:19:51,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,177 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for bebca667db35bff8ba945d067d8e9101 2024-11-07T12:19:51,185 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:19:51,186 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened bebca667db35bff8ba945d067d8e9101; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73506457, jitterRate=0.09533156454563141}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:19:51,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for bebca667db35bff8ba945d067d8e9101: 2024-11-07T12:19:51,188 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101., pid=56, masterSystemTime=1730981991148 2024-11-07T12:19:51,191 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=bebca667db35bff8ba945d067d8e9101, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:19:51,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:51,192 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:19:51,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-07T12:19:51,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure bebca667db35bff8ba945d067d8e9101, server=fbad4c8a32e4,44527,1730981867254 in 201 msec 2024-11-07T12:19:51,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T12:19:51,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=bebca667db35bff8ba945d067d8e9101, ASSIGN in 360 msec 2024-11-07T12:19:51,203 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:19:51,203 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730981991203"}]},"ts":"1730981991203"} 2024-11-07T12:19:51,206 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:19:51,207 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970136 expired clientIPAndPort=172.17.0.2:50180, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:19:51,209 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:19:51,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 493 msec 2024-11-07T12:19:51,405 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-07T12:19:51,407 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-07T12:19:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T12:19:51,726 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-07T12:19:51,726 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:19:51,726 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:51,734 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T12:19:51,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:19:51,735 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T12:19:51,753 DEBUG [hconnection-0x45f1fd4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:19:51,755 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47358, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:19:51,781 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:19:51,798 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:19:51,800 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:19:54,730 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:33401,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:19:55,812 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970131 expired clientIPAndPort=172.17.0.2:52160, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:19:55,824 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:19:56,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T12:19:56,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:56,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T12:19:56,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:19:57,814 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970129 expired clientIPAndPort=172.17.0.2:52160, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:19:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T12:19:58,816 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:20:00,803 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:20:00,803 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T12:20:00.803Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:20:00,805 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-07T12:20:00,806 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e81c4b4 to 127.0.0.1:51230 2024-11-07T12:20:00,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:00,806 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:00,806 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:00,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:00,807 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32e97093 to 127.0.0.1:51230 2024-11-07T12:20:00,827 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=229 (was 231), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=593 (was 682), ProcessCount=11 (was 11), AvailableMemoryMB=533 (was 857) 2024-11-07T12:20:00,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=593, ProcessCount=11, AvailableMemoryMB=533 2024-11-07T12:20:00,841 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x545c28c2 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c1afec0 2024-11-07T12:20:00,845 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x545c28c2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2f058d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:00,846 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x667e84fe to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d82485 2024-11-07T12:20:00,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337706a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:00,853 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:00,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T12:20:00,856 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:00,856 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:00,856 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-07T12:20:00,857 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T12:20:00,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741869_1045 (size=420) 2024-11-07T12:20:01,283 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => be85db67bdb4b8cbb0470630805ebc5d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:01,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741870_1046 (size=71) 2024-11-07T12:20:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing be85db67bdb4b8cbb0470630805ebc5d, disabling compactions & flushes 2024-11-07T12:20:01,297 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. after waiting 0 ms 2024-11-07T12:20:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,297 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for be85db67bdb4b8cbb0470630805ebc5d: 2024-11-07T12:20:01,299 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:01,299 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730982001299"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982001299"}]},"ts":"1730982001299"} 2024-11-07T12:20:01,302 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:01,305 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:01,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982001305"}]},"ts":"1730982001305"} 2024-11-07T12:20:01,307 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:01,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=be85db67bdb4b8cbb0470630805ebc5d, ASSIGN}] 2024-11-07T12:20:01,313 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=be85db67bdb4b8cbb0470630805ebc5d, ASSIGN 2024-11-07T12:20:01,314 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=be85db67bdb4b8cbb0470630805ebc5d, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:01,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=be85db67bdb4b8cbb0470630805ebc5d, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:01,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure be85db67bdb4b8cbb0470630805ebc5d, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:01,622 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:01,633 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,633 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => be85db67bdb4b8cbb0470630805ebc5d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:01,634 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,634 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:01,634 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,634 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,638 INFO [StoreOpener-be85db67bdb4b8cbb0470630805ebc5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,641 INFO [StoreOpener-be85db67bdb4b8cbb0470630805ebc5d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be85db67bdb4b8cbb0470630805ebc5d columnFamilyName testFamily 2024-11-07T12:20:01,641 DEBUG [StoreOpener-be85db67bdb4b8cbb0470630805ebc5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:01,643 INFO [StoreOpener-be85db67bdb4b8cbb0470630805ebc5d-1 {}] regionserver.HStore(327): Store=be85db67bdb4b8cbb0470630805ebc5d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:01,645 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,646 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:20:01,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:01,665 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened be85db67bdb4b8cbb0470630805ebc5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64862041, jitterRate=-0.03348027169704437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:01,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for be85db67bdb4b8cbb0470630805ebc5d: 2024-11-07T12:20:01,671 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d., pid=59, masterSystemTime=1730982001622 2024-11-07T12:20:01,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,676 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:20:01,677 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=be85db67bdb4b8cbb0470630805ebc5d, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:01,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T12:20:01,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure be85db67bdb4b8cbb0470630805ebc5d, server=fbad4c8a32e4,44527,1730981867254 in 212 msec 2024-11-07T12:20:01,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T12:20:01,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=be85db67bdb4b8cbb0470630805ebc5d, ASSIGN in 371 msec 2024-11-07T12:20:01,686 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:01,686 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982001686"}]},"ts":"1730982001686"} 2024-11-07T12:20:01,690 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:01,697 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:01,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 843 msec 2024-11-07T12:20:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T12:20:01,861 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-07T12:20:01,861 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:01,861 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:01,874 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:01,874 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:01,874 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T12:20:01,877 DEBUG [hconnection-0x16d9aa9e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:01,879 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:01,890 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:01,920 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970124 expired clientIPAndPort=172.17.0.2:52178, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:20:01,923 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:01,923 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:20:01,925 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x545c28c2 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:20:01,925 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x545c28c2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:01,927 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58654, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:20:01,930 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:20:01,931 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581550 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:01,932 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:20:01,932 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581550 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:01,933 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T12:20:03,409 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-07T12:20:04,273 ERROR [LeaseRenewer:jenkins@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:33401,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:20:05,316 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:20:06,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T12:20:06,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:07,201 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:20:10,932 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9000 2024-11-07T12:20:10,933 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x667e84fe to 127.0.0.1:51230 2024-11-07T12:20:10,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:10,933 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:10,933 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581550 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:20:10,933 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:10,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:10,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970066' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T12:20:10,933 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x545c28c2 to 127.0.0.1:51230 2024-11-07T12:20:10,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:58654 deadline: 1730982016932, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970066' does not exist 2024-11-07T12:20:10,937 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to fbad4c8a32e4,44527,1730981867254 for closing scanner id = -1672048081345970066 for be85db67bdb4b8cbb0470630805ebc5d of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T12:20:10,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=229 (was 227) - Thread LEAK? -, OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=608 (was 593) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1688 (was 533) - AvailableMemoryMB LEAK? - 2024-11-07T12:20:10,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=608, ProcessCount=11, AvailableMemoryMB=1688 2024-11-07T12:20:10,956 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07180eb4 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f68a214 2024-11-07T12:20:10,961 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x07180eb4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d97ee3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:10,962 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55e91616 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41b51657 2024-11-07T12:20:10,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fcd7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:10,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T12:20:10,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:10,974 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:10,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-07T12:20:10,976 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T12:20:10,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741871_1047 (size=431) 2024-11-07T12:20:11,394 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7dfe2b1bde36fc9612e43e4145496903, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:11,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741872_1048 (size=82) 2024-11-07T12:20:11,413 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:11,413 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 7dfe2b1bde36fc9612e43e4145496903, disabling compactions & flushes 2024-11-07T12:20:11,413 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,413 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. after waiting 0 ms 2024-11-07T12:20:11,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,414 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7dfe2b1bde36fc9612e43e4145496903: 2024-11-07T12:20:11,415 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:11,416 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1730982011415"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982011415"}]},"ts":"1730982011415"} 2024-11-07T12:20:11,418 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:11,419 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:11,420 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982011420"}]},"ts":"1730982011420"} 2024-11-07T12:20:11,422 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:11,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=7dfe2b1bde36fc9612e43e4145496903, ASSIGN}] 2024-11-07T12:20:11,429 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=7dfe2b1bde36fc9612e43e4145496903, ASSIGN 2024-11-07T12:20:11,434 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:20:11,436 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=7dfe2b1bde36fc9612e43e4145496903, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:11,587 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=7dfe2b1bde36fc9612e43e4145496903, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:11,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 7dfe2b1bde36fc9612e43e4145496903, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:11,742 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:11,749 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,750 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 7dfe2b1bde36fc9612e43e4145496903, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:11,750 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,750 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:11,750 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,751 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,753 INFO [StoreOpener-7dfe2b1bde36fc9612e43e4145496903-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,756 INFO [StoreOpener-7dfe2b1bde36fc9612e43e4145496903-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dfe2b1bde36fc9612e43e4145496903 columnFamilyName testFamily 2024-11-07T12:20:11,756 DEBUG [StoreOpener-7dfe2b1bde36fc9612e43e4145496903-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:11,757 INFO [StoreOpener-7dfe2b1bde36fc9612e43e4145496903-1 {}] regionserver.HStore(327): Store=7dfe2b1bde36fc9612e43e4145496903/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:11,758 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,759 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,762 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:20:11,766 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:11,767 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 7dfe2b1bde36fc9612e43e4145496903; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65218519, jitterRate=-0.028168335556983948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:11,768 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 7dfe2b1bde36fc9612e43e4145496903: 2024-11-07T12:20:11,769 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903., pid=62, masterSystemTime=1730982011742 2024-11-07T12:20:11,772 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,772 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:20:11,772 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=7dfe2b1bde36fc9612e43e4145496903, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:11,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T12:20:11,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 7dfe2b1bde36fc9612e43e4145496903, server=fbad4c8a32e4,44527,1730981867254 in 184 msec 2024-11-07T12:20:11,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T12:20:11,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=7dfe2b1bde36fc9612e43e4145496903, ASSIGN in 350 msec 2024-11-07T12:20:11,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:11,780 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982011780"}]},"ts":"1730982011780"} 2024-11-07T12:20:11,782 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:11,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:11,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 813 msec 2024-11-07T12:20:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T12:20:11,979 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-07T12:20:11,979 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:11,979 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:11,985 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:11,985 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:11,985 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-07T12:20:11,988 DEBUG [hconnection-0x4b1c39-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:11,989 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54374, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:11,998 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:12,001 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T12:20:12,003 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T12:20:12,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581554 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:12,004 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T12:20:12,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581554 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:12,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970062 2024-11-07T12:20:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54374 deadline: 1730982015005, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:20:12,006 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:20:12,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T12:20:12,012 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581555 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:12,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T12:20:12,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:20:12,015 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:20:12,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581556 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:12,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1672048081345970060 2024-11-07T12:20:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54374 deadline: 1730982015016, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:20:12,017 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:20:12,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:20:12,021 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581557 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:12,021 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1672048081345970059 2024-11-07T12:20:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54374 deadline: 1730982015020, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:20:12,023 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:20:12,025 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:20:12,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581558 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:12,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1672048081345970058 2024-11-07T12:20:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54374 deadline: 1730982015025, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T12:20:12,026 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:20:12,029 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T12:20:12,030 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55e91616 to 127.0.0.1:51230 2024-11-07T12:20:12,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:12,030 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:12,030 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:12,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:12,030 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x07180eb4 to 127.0.0.1:51230 2024-11-07T12:20:12,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=233 (was 229) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=608 (was 608), ProcessCount=11 (was 11), AvailableMemoryMB=1673 (was 1688) 2024-11-07T12:20:12,056 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=608, ProcessCount=11, AvailableMemoryMB=1672 2024-11-07T12:20:12,058 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d83f08c to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@146c32a6 2024-11-07T12:20:12,062 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x1d83f08c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7323ea95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:12,063 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ed1e337 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36e9ce2a 2024-11-07T12:20:12,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ec5feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:12,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T12:20:12,076 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:12,077 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:12,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-07T12:20:12,077 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T12:20:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741873_1049 (size=425) 2024-11-07T12:20:12,093 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => db5597a6c7aaebb9f2b1975a84ab1323, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:12,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741874_1050 (size=76) 2024-11-07T12:20:12,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:12,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing db5597a6c7aaebb9f2b1975a84ab1323, disabling compactions & flushes 2024-11-07T12:20:12,508 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. after waiting 0 ms 2024-11-07T12:20:12,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,508 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for db5597a6c7aaebb9f2b1975a84ab1323: 2024-11-07T12:20:12,509 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:12,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1730982012509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982012509"}]},"ts":"1730982012509"} 2024-11-07T12:20:12,511 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:12,512 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:12,512 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982012512"}]},"ts":"1730982012512"} 2024-11-07T12:20:12,514 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:12,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=db5597a6c7aaebb9f2b1975a84ab1323, ASSIGN}] 2024-11-07T12:20:12,519 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=db5597a6c7aaebb9f2b1975a84ab1323, ASSIGN 2024-11-07T12:20:12,520 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=db5597a6c7aaebb9f2b1975a84ab1323, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:12,671 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=db5597a6c7aaebb9f2b1975a84ab1323, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:12,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure db5597a6c7aaebb9f2b1975a84ab1323, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:12,826 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:12,831 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,831 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => db5597a6c7aaebb9f2b1975a84ab1323, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:12,832 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,832 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:12,832 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,832 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,836 INFO [StoreOpener-db5597a6c7aaebb9f2b1975a84ab1323-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,838 INFO [StoreOpener-db5597a6c7aaebb9f2b1975a84ab1323-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db5597a6c7aaebb9f2b1975a84ab1323 columnFamilyName testFamily 2024-11-07T12:20:12,839 DEBUG [StoreOpener-db5597a6c7aaebb9f2b1975a84ab1323-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:12,840 INFO [StoreOpener-db5597a6c7aaebb9f2b1975a84ab1323-1 {}] regionserver.HStore(327): Store=db5597a6c7aaebb9f2b1975a84ab1323/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:12,841 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,841 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,850 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:20:12,854 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:12,854 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened db5597a6c7aaebb9f2b1975a84ab1323; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61956721, jitterRate=-0.07677291333675385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:12,856 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for db5597a6c7aaebb9f2b1975a84ab1323: 2024-11-07T12:20:12,857 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323., pid=65, masterSystemTime=1730982012826 2024-11-07T12:20:12,860 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,860 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:20:12,860 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=db5597a6c7aaebb9f2b1975a84ab1323, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:12,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T12:20:12,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure db5597a6c7aaebb9f2b1975a84ab1323, server=fbad4c8a32e4,44527,1730981867254 in 189 msec 2024-11-07T12:20:12,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T12:20:12,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=db5597a6c7aaebb9f2b1975a84ab1323, ASSIGN in 347 msec 2024-11-07T12:20:12,867 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:12,868 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982012868"}]},"ts":"1730982012868"} 2024-11-07T12:20:12,870 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:12,874 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:12,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 800 msec 2024-11-07T12:20:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T12:20:13,080 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-07T12:20:13,081 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:13,081 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:13,085 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:13,085 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:13,085 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T12:20:13,088 DEBUG [hconnection-0x1e7c24d6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:13,089 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54382, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:13,101 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:13,103 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:13,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:20:13,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T12:20:14,017 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:20:14,356 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970111 expired clientIPAndPort=172.17.0.2:46568, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:20:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T12:20:15,123 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:20:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54382 deadline: 1730982016106, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:20:15,124 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:20:15,125 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581563 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:20:15,126 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2022 2024-11-07T12:20:15,128 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ed1e337 to 127.0.0.1:51230 2024-11-07T12:20:15,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:15,128 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:15,128 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:15,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:15,128 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d83f08c to 127.0.0.1:51230 2024-11-07T12:20:15,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233 (was 233), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=592 (was 608), ProcessCount=11 (was 11), AvailableMemoryMB=1634 (was 1672) 2024-11-07T12:20:15,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=592, ProcessCount=11, AvailableMemoryMB=1634 2024-11-07T12:20:15,156 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x206d77de to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33672bde 2024-11-07T12:20:15,160 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x206d77de {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acd5e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:15,161 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10062059 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a74f4f4 2024-11-07T12:20:15,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ecb5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:15,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T12:20:15,170 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:15,170 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:15,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-07T12:20:15,171 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T12:20:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741875_1051 (size=426) 2024-11-07T12:20:15,412 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-07T12:20:15,413 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-07T12:20:15,589 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8bb4504b4e674ba204befe801bad627b, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:15,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741876_1052 (size=77) 2024-11-07T12:20:15,618 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:15,618 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 8bb4504b4e674ba204befe801bad627b, disabling compactions & flushes 2024-11-07T12:20:15,618 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,618 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,618 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. after waiting 0 ms 2024-11-07T12:20:15,618 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,618 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,618 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8bb4504b4e674ba204befe801bad627b: 2024-11-07T12:20:15,620 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:15,620 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730982015620"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982015620"}]},"ts":"1730982015620"} 2024-11-07T12:20:15,622 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:15,623 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:15,623 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982015623"}]},"ts":"1730982015623"} 2024-11-07T12:20:15,625 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:15,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8bb4504b4e674ba204befe801bad627b, ASSIGN}] 2024-11-07T12:20:15,630 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8bb4504b4e674ba204befe801bad627b, ASSIGN 2024-11-07T12:20:15,632 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8bb4504b4e674ba204befe801bad627b, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:15,783 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=8bb4504b4e674ba204befe801bad627b, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:15,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 8bb4504b4e674ba204befe801bad627b, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:15,938 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:15,944 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,944 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 8bb4504b4e674ba204befe801bad627b, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:15,945 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,945 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:15,945 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,945 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,948 INFO [StoreOpener-8bb4504b4e674ba204befe801bad627b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,950 INFO [StoreOpener-8bb4504b4e674ba204befe801bad627b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bb4504b4e674ba204befe801bad627b columnFamilyName testFamily 2024-11-07T12:20:15,950 DEBUG [StoreOpener-8bb4504b4e674ba204befe801bad627b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:15,951 INFO [StoreOpener-8bb4504b4e674ba204befe801bad627b-1 {}] regionserver.HStore(327): Store=8bb4504b4e674ba204befe801bad627b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:15,953 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,953 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,957 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 8bb4504b4e674ba204befe801bad627b 2024-11-07T12:20:15,963 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:15,964 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 8bb4504b4e674ba204befe801bad627b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64344561, jitterRate=-0.04119132459163666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:15,965 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 8bb4504b4e674ba204befe801bad627b: 2024-11-07T12:20:15,967 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., pid=68, masterSystemTime=1730982015938 2024-11-07T12:20:15,969 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=8bb4504b4e674ba204befe801bad627b, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:15,970 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,970 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:20:15,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-07T12:20:15,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 8bb4504b4e674ba204befe801bad627b, server=fbad4c8a32e4,44527,1730981867254 in 187 msec 2024-11-07T12:20:15,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T12:20:15,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8bb4504b4e674ba204befe801bad627b, ASSIGN in 345 msec 2024-11-07T12:20:15,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:15,977 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982015977"}]},"ts":"1730982015977"} 2024-11-07T12:20:15,979 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:15,982 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:15,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 814 msec 2024-11-07T12:20:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T12:20:16,174 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-07T12:20:16,174 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:16,174 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:16,179 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:16,180 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:16,180 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T12:20:16,185 DEBUG [hconnection-0x43e3d244-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:16,186 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:16,196 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:16,222 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:20:16,234 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:20:16,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T12:20:16,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:16,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T12:20:16,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:16,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T12:20:16,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:17,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:20:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T12:20:18,255 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:20:19,244 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:20:19,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T12:20:19.244Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:20:19,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3023 2024-11-07T12:20:19,248 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10062059 to 127.0.0.1:51230 2024-11-07T12:20:19,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:19,249 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:19,249 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:19,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:19,249 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x206d77de to 127.0.0.1:51230 2024-11-07T12:20:19,280 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=232 (was 233), OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=592 (was 592), ProcessCount=11 (was 11), AvailableMemoryMB=1692 (was 1634) - AvailableMemoryMB LEAK? - 2024-11-07T12:20:19,317 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=232, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=592, ProcessCount=11, AvailableMemoryMB=1692 2024-11-07T12:20:19,319 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4044be19 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@689afe29 2024-11-07T12:20:19,347 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x4044be19 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bc317a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:19,350 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4275ec03 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@de35236 2024-11-07T12:20:19,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683291e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:19,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T12:20:19,383 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:19,383 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:19,384 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:19,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-07T12:20:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T12:20:19,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741877_1053 (size=424) 2024-11-07T12:20:19,755 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:20:19,859 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bcdda0a06bb9bc89931546d07c0b88ed, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:19,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741878_1054 (size=75) 2024-11-07T12:20:19,910 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:19,911 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing bcdda0a06bb9bc89931546d07c0b88ed, disabling compactions & flushes 2024-11-07T12:20:19,911 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:19,911 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:19,911 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. after waiting 0 ms 2024-11-07T12:20:19,911 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:19,911 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:19,911 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for bcdda0a06bb9bc89931546d07c0b88ed: 2024-11-07T12:20:19,912 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:19,913 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1730982019913"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982019913"}]},"ts":"1730982019913"} 2024-11-07T12:20:19,915 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:19,917 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:19,917 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982019917"}]},"ts":"1730982019917"} 2024-11-07T12:20:19,919 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:19,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bcdda0a06bb9bc89931546d07c0b88ed, ASSIGN}] 2024-11-07T12:20:19,960 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bcdda0a06bb9bc89931546d07c0b88ed, ASSIGN 2024-11-07T12:20:19,969 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bcdda0a06bb9bc89931546d07c0b88ed, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:20,124 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=bcdda0a06bb9bc89931546d07c0b88ed, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:20,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure bcdda0a06bb9bc89931546d07c0b88ed, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:20,278 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:20,289 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:20,289 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => bcdda0a06bb9bc89931546d07c0b88ed, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:20,290 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,290 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:20,290 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,290 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,294 INFO [StoreOpener-bcdda0a06bb9bc89931546d07c0b88ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,298 INFO [StoreOpener-bcdda0a06bb9bc89931546d07c0b88ed-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcdda0a06bb9bc89931546d07c0b88ed columnFamilyName testFamily 2024-11-07T12:20:20,299 DEBUG [StoreOpener-bcdda0a06bb9bc89931546d07c0b88ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:20,299 INFO [StoreOpener-bcdda0a06bb9bc89931546d07c0b88ed-1 {}] regionserver.HStore(327): Store=bcdda0a06bb9bc89931546d07c0b88ed/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:20,301 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,302 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,305 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:20:20,321 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:20,327 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened bcdda0a06bb9bc89931546d07c0b88ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74927262, jitterRate=0.11650320887565613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:20,328 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for bcdda0a06bb9bc89931546d07c0b88ed: 2024-11-07T12:20:20,330 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed., pid=71, masterSystemTime=1730982020278 2024-11-07T12:20:20,332 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:20,332 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:20:20,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=bcdda0a06bb9bc89931546d07c0b88ed, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:20,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T12:20:20,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure bcdda0a06bb9bc89931546d07c0b88ed, server=fbad4c8a32e4,44527,1730981867254 in 208 msec 2024-11-07T12:20:20,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-07T12:20:20,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=bcdda0a06bb9bc89931546d07c0b88ed, ASSIGN in 389 msec 2024-11-07T12:20:20,340 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:20,340 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982020340"}]},"ts":"1730982020340"} 2024-11-07T12:20:20,346 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:20,350 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:20,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 970 msec 2024-11-07T12:20:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T12:20:20,390 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-07T12:20:20,390 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:20,390 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:20,396 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:20,396 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:20,396 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T12:20:20,399 DEBUG [hconnection-0x513a3c03-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:20,400 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40290, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:20,410 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:20,412 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:20,412 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:20:20,414 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x4044be19 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:20:20,414 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x4044be19 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:20,415 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40292, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:20:20,418 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:20:20,418 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581575 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:20,419 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:20:20,419 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581575 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:20,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T12:20:21,415 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-07T12:20:21,416 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-07T12:20:23,423 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3003 2024-11-07T12:20:23,423 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4275ec03 to 127.0.0.1:51230 2024-11-07T12:20:23,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:23,423 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:23,423 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581575 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:20:23,423 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:23,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:23,424 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4044be19 to 127.0.0.1:51230 2024-11-07T12:20:23,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970041' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T12:20:23,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:40292 deadline: 1730982024423, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970041' does not exist 2024-11-07T12:20:23,427 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to fbad4c8a32e4,44527,1730981867254 for closing scanner id = -1672048081345970041 for bcdda0a06bb9bc89931546d07c0b88ed of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1672048081345970041' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:20:23,437 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231 (was 232), OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=584 (was 592), ProcessCount=11 (was 11), AvailableMemoryMB=2356 (was 1692) - AvailableMemoryMB LEAK? - 2024-11-07T12:20:23,449 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=584, ProcessCount=11, AvailableMemoryMB=2353 2024-11-07T12:20:23,450 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x694e2010 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31ebe2d 2024-11-07T12:20:23,454 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x694e2010 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a050b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:23,455 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x160d069d to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f92b4da 2024-11-07T12:20:23,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17444b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:23,461 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:23,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-07T12:20:23,469 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:23,470 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:23,470 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-07T12:20:23,471 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T12:20:23,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741879_1055 (size=416) 2024-11-07T12:20:23,505 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0b0d536521b4033814c4e766b23cf3ae, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:23,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741880_1056 (size=67) 2024-11-07T12:20:23,916 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:23,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 0b0d536521b4033814c4e766b23cf3ae, disabling compactions & flushes 2024-11-07T12:20:23,917 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:23,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:23,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. after waiting 0 ms 2024-11-07T12:20:23,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:23,917 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:23,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0b0d536521b4033814c4e766b23cf3ae: 2024-11-07T12:20:23,918 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:23,918 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1730982023918"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982023918"}]},"ts":"1730982023918"} 2024-11-07T12:20:23,920 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:20:23,921 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:23,922 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:23,922 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982023922"}]},"ts":"1730982023922"} 2024-11-07T12:20:23,923 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:23,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b0d536521b4033814c4e766b23cf3ae, ASSIGN}] 2024-11-07T12:20:23,929 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b0d536521b4033814c4e766b23cf3ae, ASSIGN 2024-11-07T12:20:23,930 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b0d536521b4033814c4e766b23cf3ae, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:23,984 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970107 expired clientIPAndPort=172.17.0.2:46572, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:20:24,080 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0b0d536521b4033814c4e766b23cf3ae, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:24,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 0b0d536521b4033814c4e766b23cf3ae, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:24,235 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:24,244 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:24,244 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 0b0d536521b4033814c4e766b23cf3ae, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:24,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:24,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,258 INFO [StoreOpener-0b0d536521b4033814c4e766b23cf3ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,266 INFO [StoreOpener-0b0d536521b4033814c4e766b23cf3ae-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b0d536521b4033814c4e766b23cf3ae columnFamilyName testFamily 2024-11-07T12:20:24,266 DEBUG [StoreOpener-0b0d536521b4033814c4e766b23cf3ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:24,267 INFO [StoreOpener-0b0d536521b4033814c4e766b23cf3ae-1 {}] regionserver.HStore(327): Store=0b0d536521b4033814c4e766b23cf3ae/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:24,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,271 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:20:24,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:24,280 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 0b0d536521b4033814c4e766b23cf3ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62825915, jitterRate=-0.06382091343402863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:24,281 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 0b0d536521b4033814c4e766b23cf3ae: 2024-11-07T12:20:24,284 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae., pid=74, masterSystemTime=1730982024234 2024-11-07T12:20:24,288 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0b0d536521b4033814c4e766b23cf3ae, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:24,288 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:24,288 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:20:24,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-07T12:20:24,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 0b0d536521b4033814c4e766b23cf3ae, server=fbad4c8a32e4,44527,1730981867254 in 217 msec 2024-11-07T12:20:24,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T12:20:24,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0b0d536521b4033814c4e766b23cf3ae, ASSIGN in 374 msec 2024-11-07T12:20:24,304 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:24,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982024304"}]},"ts":"1730982024304"} 2024-11-07T12:20:24,306 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:24,316 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:24,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 855 msec 2024-11-07T12:20:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T12:20:24,477 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-07T12:20:24,477 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:24,477 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:24,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-07T12:20:24,486 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581576 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:24,487 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:24,488 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:24,488 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-07T12:20:24,491 DEBUG [hconnection-0x3a4a1c9b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:24,492 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40294, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:24,501 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:24,509 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:24,515 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581579 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:20:24,515 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T12:20:25,792 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:20:26,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T12:20:26,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T12:20:26,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:27,417 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-07T12:20:31,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6506 2024-11-07T12:20:31,016 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x160d069d to 127.0.0.1:51230 2024-11-07T12:20:31,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:31,016 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:31,016 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:31,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:31,016 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x694e2010 to 127.0.0.1:51230 2024-11-07T12:20:31,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230 (was 231), OpenFileDescriptor=452 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=563 (was 584), ProcessCount=11 (was 11), AvailableMemoryMB=3100 (was 2353) - AvailableMemoryMB LEAK? - 2024-11-07T12:20:31,035 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=563, ProcessCount=11, AvailableMemoryMB=3100 2024-11-07T12:20:31,037 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a4755e9 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13821ae2 2024-11-07T12:20:31,044 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x4a4755e9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75db7d3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:31,045 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6782cdc0 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36af503d 2024-11-07T12:20:31,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b7b31c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:31,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-07T12:20:31,056 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:31,056 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:31,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-07T12:20:31,057 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T12:20:31,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741881_1057 (size=420) 2024-11-07T12:20:31,479 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4bf92c6c7d15fd30800845193778cf48, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:31,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741882_1058 (size=71) 2024-11-07T12:20:31,670 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:33401,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:20:31,802 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970101 expired clientIPAndPort=172.17.0.2:54562, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:20:31,887 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:31,887 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 4bf92c6c7d15fd30800845193778cf48, disabling compactions & flushes 2024-11-07T12:20:31,887 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:31,887 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:31,887 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. after waiting 0 ms 2024-11-07T12:20:31,887 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:31,887 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:31,887 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4bf92c6c7d15fd30800845193778cf48: 2024-11-07T12:20:31,888 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:31,889 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730982031888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982031888"}]},"ts":"1730982031888"} 2024-11-07T12:20:31,890 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:31,892 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:31,892 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982031892"}]},"ts":"1730982031892"} 2024-11-07T12:20:31,893 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:31,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=4bf92c6c7d15fd30800845193778cf48, ASSIGN}] 2024-11-07T12:20:31,898 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=4bf92c6c7d15fd30800845193778cf48, ASSIGN 2024-11-07T12:20:31,899 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=4bf92c6c7d15fd30800845193778cf48, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:32,049 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=4bf92c6c7d15fd30800845193778cf48, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:32,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 4bf92c6c7d15fd30800845193778cf48, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T12:20:32,204 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:32,211 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:32,212 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => 4bf92c6c7d15fd30800845193778cf48, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:32,212 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,212 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:32,212 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for 4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,213 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for 4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,219 INFO [StoreOpener-4bf92c6c7d15fd30800845193778cf48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,221 INFO [StoreOpener-4bf92c6c7d15fd30800845193778cf48-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bf92c6c7d15fd30800845193778cf48 columnFamilyName testFamily 2024-11-07T12:20:32,221 DEBUG [StoreOpener-4bf92c6c7d15fd30800845193778cf48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:32,222 INFO [StoreOpener-4bf92c6c7d15fd30800845193778cf48-1 {}] regionserver.HStore(327): Store=4bf92c6c7d15fd30800845193778cf48/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:32,223 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,224 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,227 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for 4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:20:32,231 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:32,232 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened 4bf92c6c7d15fd30800845193778cf48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63772179, jitterRate=-0.04972048103809357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:32,233 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for 4bf92c6c7d15fd30800845193778cf48: 2024-11-07T12:20:32,234 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48., pid=77, masterSystemTime=1730982032203 2024-11-07T12:20:32,237 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:32,237 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:20:32,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=4bf92c6c7d15fd30800845193778cf48, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:32,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T12:20:32,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 4bf92c6c7d15fd30800845193778cf48, server=fbad4c8a32e4,44527,1730981867254 in 187 msec 2024-11-07T12:20:32,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-07T12:20:32,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=4bf92c6c7d15fd30800845193778cf48, ASSIGN in 344 msec 2024-11-07T12:20:32,243 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:32,243 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982032243"}]},"ts":"1730982032243"} 2024-11-07T12:20:32,245 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:32,249 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:32,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1950 sec 2024-11-07T12:20:33,419 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-07T12:20:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T12:20:34,061 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-07T12:20:34,061 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:34,061 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:34,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-07T12:20:34,066 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581580 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:34,067 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:34,067 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:34,067 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-07T12:20:34,069 DEBUG [hconnection-0x49c09a6a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:34,070 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36862, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:34,077 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:34,080 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:34,081 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581583 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T12:20:34,082 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T12:20:35,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-07T12:20:35,583 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6782cdc0 to 127.0.0.1:51230 2024-11-07T12:20:35,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:35,583 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:35,583 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:35,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:35,583 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a4755e9 to 127.0.0.1:51230 2024-11-07T12:20:35,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 230) - Thread LEAK? -, OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 563), ProcessCount=11 (was 11), AvailableMemoryMB=2730 (was 3100) 2024-11-07T12:20:35,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=2729 2024-11-07T12:20:35,602 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b3d660d to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28c20935 2024-11-07T12:20:35,606 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x7b3d660d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308621c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:35,607 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55ac4d45 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@512565d3 2024-11-07T12:20:35,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41387cf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:35,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-07T12:20:35,615 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:35,615 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:35,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-07T12:20:35,615 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T12:20:35,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741883_1059 (size=419) 2024-11-07T12:20:36,026 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 96acfee49f6cde18af501f86ea60e0a4, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:36,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741884_1060 (size=70) 2024-11-07T12:20:36,435 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:36,435 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 96acfee49f6cde18af501f86ea60e0a4, disabling compactions & flushes 2024-11-07T12:20:36,435 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,435 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,435 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. after waiting 0 ms 2024-11-07T12:20:36,435 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,435 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,435 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 96acfee49f6cde18af501f86ea60e0a4: 2024-11-07T12:20:36,436 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:36,436 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1730982036436"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982036436"}]},"ts":"1730982036436"} 2024-11-07T12:20:36,438 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:36,439 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:36,439 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982036439"}]},"ts":"1730982036439"} 2024-11-07T12:20:36,441 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:36,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=96acfee49f6cde18af501f86ea60e0a4, ASSIGN}] 2024-11-07T12:20:36,446 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=96acfee49f6cde18af501f86ea60e0a4, ASSIGN 2024-11-07T12:20:36,447 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=96acfee49f6cde18af501f86ea60e0a4, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:36,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-07T12:20:36,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:36,598 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=96acfee49f6cde18af501f86ea60e0a4, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:36,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 96acfee49f6cde18af501f86ea60e0a4, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T12:20:36,752 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:36,757 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,758 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => 96acfee49f6cde18af501f86ea60e0a4, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:36,758 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,758 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:36,758 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for 96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,758 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for 96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,760 INFO [StoreOpener-96acfee49f6cde18af501f86ea60e0a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,762 INFO [StoreOpener-96acfee49f6cde18af501f86ea60e0a4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96acfee49f6cde18af501f86ea60e0a4 columnFamilyName testFamily 2024-11-07T12:20:36,763 DEBUG [StoreOpener-96acfee49f6cde18af501f86ea60e0a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:36,763 INFO [StoreOpener-96acfee49f6cde18af501f86ea60e0a4-1 {}] regionserver.HStore(327): Store=96acfee49f6cde18af501f86ea60e0a4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:36,764 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,765 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,768 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for 96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:20:36,771 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:36,772 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened 96acfee49f6cde18af501f86ea60e0a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64787261, jitterRate=-0.0345945805311203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:36,773 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for 96acfee49f6cde18af501f86ea60e0a4: 2024-11-07T12:20:36,774 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4., pid=80, masterSystemTime=1730982036752 2024-11-07T12:20:36,776 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,776 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:20:36,777 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=96acfee49f6cde18af501f86ea60e0a4, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:36,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-07T12:20:36,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 96acfee49f6cde18af501f86ea60e0a4, server=fbad4c8a32e4,44527,1730981867254 in 179 msec 2024-11-07T12:20:36,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T12:20:36,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=96acfee49f6cde18af501f86ea60e0a4, ASSIGN in 335 msec 2024-11-07T12:20:36,782 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:36,782 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982036782"}]},"ts":"1730982036782"} 2024-11-07T12:20:36,784 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:36,787 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:36,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1740 sec 2024-11-07T12:20:37,774 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:20:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T12:20:38,620 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-07T12:20:38,621 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:38,621 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:38,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-07T12:20:38,625 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581584 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:38,627 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:38,627 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:38,627 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-07T12:20:38,629 DEBUG [hconnection-0x158c11f5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:38,630 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36876, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:38,637 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:38,639 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:38,640 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:20:38,640 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:38,641 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:20:38,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581587 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:38,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T12:20:38,809 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970099 expired clientIPAndPort=172.17.0.2:54562, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:20:39,420 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-07T12:20:40,986 ERROR [LeaseRenewer:jenkins@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:33401,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:20:41,642 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4008: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3001ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4008: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3001ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3001ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3001ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:20:41,642 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3001 2024-11-07T12:20:41,643 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55ac4d45 to 127.0.0.1:51230 2024-11-07T12:20:41,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:41,643 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:41,643 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:41,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:41,643 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b3d660d to 127.0.0.1:51230 2024-11-07T12:20:41,653 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=229 (was 233), OpenFileDescriptor=451 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=492 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=2376 (was 2729) 2024-11-07T12:20:41,663 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=229, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=492, ProcessCount=11, AvailableMemoryMB=2376 2024-11-07T12:20:41,665 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b40e307 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7675fd68 2024-11-07T12:20:41,669 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x1b40e307 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d4a998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:41,670 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3319f4ca to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31d9b463 2024-11-07T12:20:41,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d3184, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:41,677 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:41,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T12:20:41,679 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:41,679 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:41,679 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-07T12:20:41,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T12:20:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741885_1061 (size=421) 2024-11-07T12:20:42,091 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 66173e428a30393976f3b6c0f80d3be0, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:42,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741886_1062 (size=72) 2024-11-07T12:20:42,142 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:20:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 66173e428a30393976f3b6c0f80d3be0, disabling compactions & flushes 2024-11-07T12:20:42,500 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. after waiting 0 ms 2024-11-07T12:20:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,500 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 66173e428a30393976f3b6c0f80d3be0: 2024-11-07T12:20:42,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:42,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730982042502"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982042502"}]},"ts":"1730982042502"} 2024-11-07T12:20:42,504 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:42,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:42,505 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982042505"}]},"ts":"1730982042505"} 2024-11-07T12:20:42,506 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:42,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66173e428a30393976f3b6c0f80d3be0, ASSIGN}] 2024-11-07T12:20:42,517 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66173e428a30393976f3b6c0f80d3be0, ASSIGN 2024-11-07T12:20:42,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66173e428a30393976f3b6c0f80d3be0, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:42,668 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=66173e428a30393976f3b6c0f80d3be0, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:42,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 66173e428a30393976f3b6c0f80d3be0, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T12:20:42,822 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:42,827 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => 66173e428a30393976f3b6c0f80d3be0, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:42,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:42,828 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for 66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,828 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for 66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,829 INFO [StoreOpener-66173e428a30393976f3b6c0f80d3be0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,831 INFO [StoreOpener-66173e428a30393976f3b6c0f80d3be0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66173e428a30393976f3b6c0f80d3be0 columnFamilyName testFamily 2024-11-07T12:20:42,831 DEBUG [StoreOpener-66173e428a30393976f3b6c0f80d3be0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:42,832 INFO [StoreOpener-66173e428a30393976f3b6c0f80d3be0-1 {}] regionserver.HStore(327): Store=66173e428a30393976f3b6c0f80d3be0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:42,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for 66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:20:42,839 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:42,839 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened 66173e428a30393976f3b6c0f80d3be0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59083333, jitterRate=-0.11958973109722137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:42,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for 66173e428a30393976f3b6c0f80d3be0: 2024-11-07T12:20:42,841 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0., pid=83, masterSystemTime=1730982042822 2024-11-07T12:20:42,843 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,843 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:20:42,843 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=66173e428a30393976f3b6c0f80d3be0, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:42,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T12:20:42,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 66173e428a30393976f3b6c0f80d3be0, server=fbad4c8a32e4,44527,1730981867254 in 175 msec 2024-11-07T12:20:42,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-07T12:20:42,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=66173e428a30393976f3b6c0f80d3be0, ASSIGN in 331 msec 2024-11-07T12:20:42,848 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:42,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982042848"}]},"ts":"1730982042848"} 2024-11-07T12:20:42,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:42,852 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:42,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1750 sec 2024-11-07T12:20:43,913 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970094 expired clientIPAndPort=172.17.0.2:54004, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:20:44,017 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:20:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T12:20:44,683 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-07T12:20:44,683 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:44,684 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:44,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-07T12:20:44,689 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581588 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:44,691 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:44,691 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:44,691 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T12:20:44,693 DEBUG [hconnection-0x148bafb1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:44,695 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58102, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:44,704 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:44,710 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:20:44,715 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:20:44,715 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T12:20:45,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-07T12:20:46,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T12:20:46,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:46,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T12:20:46,581 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:48,017 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970089 expired clientIPAndPort=172.17.0.2:33816, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:20:48,374 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:20:50,019 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970087 expired clientIPAndPort=172.17.0.2:33816, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:20:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T12:20:51,727 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:20:51,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:58102 deadline: 1730982053714, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T12:20:51,728 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T12:20:51,729 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16774695992363581591 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T12:20:51,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7042 2024-11-07T12:20:51,753 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3319f4ca to 127.0.0.1:51230 2024-11-07T12:20:51,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:51,753 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:20:51,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:20:51,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:20:51,753 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b40e307 to 127.0.0.1:51230 2024-11-07T12:20:51,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=229 (was 229), OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 492), ProcessCount=11 (was 11), AvailableMemoryMB=2192 (was 2376) 2024-11-07T12:20:51,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=2192 2024-11-07T12:20:51,773 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35edb7ba to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@154c1bac 2024-11-07T12:20:51,777 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x35edb7ba {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a4865a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:51,778 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x18d16993 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8825c2c 2024-11-07T12:20:51,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500a6ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:20:51,784 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:20:51,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T12:20:51,786 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:20:51,786 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:51,786 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-07T12:20:51,787 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:20:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T12:20:51,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741887_1063 (size=427) 2024-11-07T12:20:52,196 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6497436b1160bd1dd303db7791f75ae1, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:20:52,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741888_1064 (size=78) 2024-11-07T12:20:52,603 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:52,604 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 6497436b1160bd1dd303db7791f75ae1, disabling compactions & flushes 2024-11-07T12:20:52,604 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,604 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,604 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. after waiting 0 ms 2024-11-07T12:20:52,604 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,604 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,604 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6497436b1160bd1dd303db7791f75ae1: 2024-11-07T12:20:52,605 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:20:52,605 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1730982052605"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982052605"}]},"ts":"1730982052605"} 2024-11-07T12:20:52,607 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:20:52,608 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:20:52,608 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982052608"}]},"ts":"1730982052608"} 2024-11-07T12:20:52,609 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:20:52,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=6497436b1160bd1dd303db7791f75ae1, ASSIGN}] 2024-11-07T12:20:52,613 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=6497436b1160bd1dd303db7791f75ae1, ASSIGN 2024-11-07T12:20:52,614 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=6497436b1160bd1dd303db7791f75ae1, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:20:52,765 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=6497436b1160bd1dd303db7791f75ae1, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:52,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure 6497436b1160bd1dd303db7791f75ae1, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:20:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T12:20:52,918 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:52,924 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => 6497436b1160bd1dd303db7791f75ae1, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:20:52,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:20:52,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,927 INFO [StoreOpener-6497436b1160bd1dd303db7791f75ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,929 INFO [StoreOpener-6497436b1160bd1dd303db7791f75ae1-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6497436b1160bd1dd303db7791f75ae1 columnFamilyName testFamily 2024-11-07T12:20:52,929 DEBUG [StoreOpener-6497436b1160bd1dd303db7791f75ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:20:52,929 INFO [StoreOpener-6497436b1160bd1dd303db7791f75ae1-1 {}] regionserver.HStore(327): Store=6497436b1160bd1dd303db7791f75ae1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:20:52,930 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,931 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:20:52,935 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:20:52,936 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened 6497436b1160bd1dd303db7791f75ae1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61787521, jitterRate=-0.07929418981075287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:20:52,936 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for 6497436b1160bd1dd303db7791f75ae1: 2024-11-07T12:20:52,937 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., pid=86, masterSystemTime=1730982052918 2024-11-07T12:20:52,939 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,939 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:20:52,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=6497436b1160bd1dd303db7791f75ae1, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:20:52,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-07T12:20:52,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure 6497436b1160bd1dd303db7791f75ae1, server=fbad4c8a32e4,44527,1730981867254 in 174 msec 2024-11-07T12:20:52,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T12:20:52,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=6497436b1160bd1dd303db7791f75ae1, ASSIGN in 330 msec 2024-11-07T12:20:52,944 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:20:52,944 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982052944"}]},"ts":"1730982052944"} 2024-11-07T12:20:52,946 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:20:52,948 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:20:52,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1640 sec 2024-11-07T12:20:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T12:20:54,790 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-07T12:20:54,790 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:20:54,790 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:54,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-07T12:20:54,794 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581593 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:20:54,795 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T12:20:54,795 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:20:54,796 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T12:20:54,798 DEBUG [hconnection-0x7e2909c9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:54,799 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42840, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:54,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:20:54,808 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:20:54,808 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:20:54,810 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x35edb7ba {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:20:54,810 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x35edb7ba {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:20:54,811 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42850, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:20:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:20:54,813 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:20:56,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T12:20:56,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:20:57,424 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T12:20:58,326 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970073 expired clientIPAndPort=172.17.0.2:47358, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:20:58,427 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:21:00,823 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-07T12:21:00,823 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-07T12:21:00,823 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:21:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T12:21:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:21:01,842 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T12:21:02,233 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:33401,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:21:03,813 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1970ms, rpcTimeout=1966ms 2024-11-07T12:21:03,813 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1970ms, rpcTimeout=1966ms 2024-11-07T12:21:03,813 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:21:03,814 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T12:21:00.823Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-07T12:21:03.813Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1970ms, rpcTimeout=1966ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1970ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1970ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T12:21:03,815 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-07T12:21:03,815 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x18d16993 to 127.0.0.1:51230 2024-11-07T12:21:03,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:03,815 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:21:03,815 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:21:03,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:03,815 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x35edb7ba to 127.0.0.1:51230 2024-11-07T12:21:03,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=224 (was 229), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=2032 (was 2192) 2024-11-07T12:21:03,835 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=224, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=2032 2024-11-07T12:21:03,837 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73bdd438 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a339bf3 2024-11-07T12:21:03,843 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x73bdd438 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e22752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:21:03,844 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6054f027 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1d9612bd 2024-11-07T12:21:03,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170923, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:21:03,852 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:21:03,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-07T12:21:03,853 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:21:03,854 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:21:03,854 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-07T12:21:03,854 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:21:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T12:21:03,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741889_1065 (size=415) 2024-11-07T12:21:04,268 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bc6ca4f162ccd89027841a9df4cc6af5, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:21:04,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741890_1066 (size=66) 2024-11-07T12:21:04,676 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:21:04,676 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing bc6ca4f162ccd89027841a9df4cc6af5, disabling compactions & flushes 2024-11-07T12:21:04,676 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:04,676 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:04,676 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. after waiting 0 ms 2024-11-07T12:21:04,676 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:04,676 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:04,676 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for bc6ca4f162ccd89027841a9df4cc6af5: 2024-11-07T12:21:04,677 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:21:04,678 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1730982064677"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982064677"}]},"ts":"1730982064677"} 2024-11-07T12:21:04,679 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:21:04,680 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:21:04,680 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982064680"}]},"ts":"1730982064680"} 2024-11-07T12:21:04,681 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:21:04,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=bc6ca4f162ccd89027841a9df4cc6af5, ASSIGN}] 2024-11-07T12:21:04,686 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=bc6ca4f162ccd89027841a9df4cc6af5, ASSIGN 2024-11-07T12:21:04,687 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=bc6ca4f162ccd89027841a9df4cc6af5, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:21:04,837 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=bc6ca4f162ccd89027841a9df4cc6af5, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:04,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure bc6ca4f162ccd89027841a9df4cc6af5, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:21:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T12:21:04,991 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:04,997 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:04,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => bc6ca4f162ccd89027841a9df4cc6af5, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:21:04,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:04,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:21:04,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:04,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:04,999 INFO [StoreOpener-bc6ca4f162ccd89027841a9df4cc6af5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:05,000 INFO [StoreOpener-bc6ca4f162ccd89027841a9df4cc6af5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc6ca4f162ccd89027841a9df4cc6af5 columnFamilyName testFamily 2024-11-07T12:21:05,001 DEBUG [StoreOpener-bc6ca4f162ccd89027841a9df4cc6af5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:21:05,001 INFO [StoreOpener-bc6ca4f162ccd89027841a9df4cc6af5-1 {}] regionserver.HStore(327): Store=bc6ca4f162ccd89027841a9df4cc6af5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:21:05,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:05,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:05,005 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:05,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:21:05,008 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened bc6ca4f162ccd89027841a9df4cc6af5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61521266, jitterRate=-0.08326169848442078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:21:05,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for bc6ca4f162ccd89027841a9df4cc6af5: 2024-11-07T12:21:05,009 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5., pid=89, masterSystemTime=1730982064991 2024-11-07T12:21:05,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:05,011 INFO [RS_OPEN_PRIORITY_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:05,012 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=bc6ca4f162ccd89027841a9df4cc6af5, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:05,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T12:21:05,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure bc6ca4f162ccd89027841a9df4cc6af5, server=fbad4c8a32e4,44527,1730981867254 in 174 msec 2024-11-07T12:21:05,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T12:21:05,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=bc6ca4f162ccd89027841a9df4cc6af5, ASSIGN in 330 msec 2024-11-07T12:21:05,017 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:21:05,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982065017"}]},"ts":"1730982065017"} 2024-11-07T12:21:05,018 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:21:05,021 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:21:05,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1690 sec 2024-11-07T12:21:05,332 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970071 expired clientIPAndPort=172.17.0.2:47358, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:21:06,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T12:21:06,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:21:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T12:21:06,858 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-07T12:21:06,858 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:21:06,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:21:06,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-07T12:21:06,862 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581600 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:21:06,863 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T12:21:06,864 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:21:06,864 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-07T12:21:06,866 DEBUG [hconnection-0x3d75170a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:21:06,867 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35960, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:21:06,874 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:21:06,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T12:21:06,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T12:21:06,878 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581603 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:21:06,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T12:21:06,879 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581603 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:21:06,880 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T12:21:08,343 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:21:09,425 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-07T12:21:10,537 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:21:11,438 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970066 expired clientIPAndPort=172.17.0.2:58654, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:21:11,685 ERROR [LeaseRenewer:jenkins@localhost:33401 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:33401,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:21:12,038 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970059 expired clientIPAndPort=172.17.0.2:54374, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:12,039 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970060 expired clientIPAndPort=172.17.0.2:54374, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:12,039 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970062 expired clientIPAndPort=172.17.0.2:54374, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:14,017 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:21:15,141 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970053 expired clientIPAndPort=172.17.0.2:54382, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:21:15,883 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10006: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9003ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10006: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9003ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9003ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9003ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T12:21:15,883 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9004 2024-11-07T12:21:15,883 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6054f027 to 127.0.0.1:51230 2024-11-07T12:21:15,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:15,884 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:21:15,884 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:21:15,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:15,884 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x73bdd438 to 127.0.0.1:51230 2024-11-07T12:21:15,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=225 (was 224) - Thread LEAK? -, OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=338 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=1999 (was 2032) 2024-11-07T12:21:15,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=225, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=1998 2024-11-07T12:21:15,906 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x109e8270 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2554bb0a 2024-11-07T12:21:15,910 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x109e8270 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@976b9f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:21:15,911 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bf05785 to 127.0.0.1:51230 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48ceebd8 2024-11-07T12:21:15,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca77db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:21:15,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:21:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T12:21:15,918 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:21:15,918 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:21:15,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-07T12:21:15,918 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:21:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T12:21:15,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741891_1067 (size=431) 2024-11-07T12:21:16,329 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5c909e1f401dd618f9fe9ecea4bdcfbc, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc 2024-11-07T12:21:16,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741892_1068 (size=82) 2024-11-07T12:21:16,380 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:21:16,738 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:21:16,739 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 5c909e1f401dd618f9fe9ecea4bdcfbc, disabling compactions & flushes 2024-11-07T12:21:16,739 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:16,739 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:16,739 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. after waiting 0 ms 2024-11-07T12:21:16,739 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:16,739 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:16,739 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5c909e1f401dd618f9fe9ecea4bdcfbc: 2024-11-07T12:21:16,740 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:21:16,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1730982076740"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982076740"}]},"ts":"1730982076740"} 2024-11-07T12:21:16,742 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T12:21:16,742 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:21:16,742 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982076742"}]},"ts":"1730982076742"} 2024-11-07T12:21:16,743 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T12:21:16,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=5c909e1f401dd618f9fe9ecea4bdcfbc, ASSIGN}] 2024-11-07T12:21:16,748 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=5c909e1f401dd618f9fe9ecea4bdcfbc, ASSIGN 2024-11-07T12:21:16,749 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=5c909e1f401dd618f9fe9ecea4bdcfbc, ASSIGN; state=OFFLINE, location=fbad4c8a32e4,44527,1730981867254; forceNewPlan=false, retain=false 2024-11-07T12:21:16,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=5c909e1f401dd618f9fe9ecea4bdcfbc, regionState=OPENING, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:16,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 5c909e1f401dd618f9fe9ecea4bdcfbc, server=fbad4c8a32e4,44527,1730981867254}] 2024-11-07T12:21:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T12:21:17,054 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:17,059 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:17,059 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 5c909e1f401dd618f9fe9ecea4bdcfbc, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:21:17,059 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,059 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:21:17,060 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,060 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,062 INFO [StoreOpener-5c909e1f401dd618f9fe9ecea4bdcfbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,063 INFO [StoreOpener-5c909e1f401dd618f9fe9ecea4bdcfbc-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c909e1f401dd618f9fe9ecea4bdcfbc columnFamilyName testFamily 2024-11-07T12:21:17,063 DEBUG [StoreOpener-5c909e1f401dd618f9fe9ecea4bdcfbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:21:17,064 INFO [StoreOpener-5c909e1f401dd618f9fe9ecea4bdcfbc-1 {}] regionserver.HStore(327): Store=5c909e1f401dd618f9fe9ecea4bdcfbc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:21:17,065 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,065 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,067 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:17,069 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:21:17,069 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 5c909e1f401dd618f9fe9ecea4bdcfbc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65918293, jitterRate=-0.017740890383720398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T12:21:17,070 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 5c909e1f401dd618f9fe9ecea4bdcfbc: 2024-11-07T12:21:17,071 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., pid=92, masterSystemTime=1730982077053 2024-11-07T12:21:17,072 DEBUG [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:17,072 INFO [RS_OPEN_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:17,073 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=5c909e1f401dd618f9fe9ecea4bdcfbc, regionState=OPEN, openSeqNum=2, regionLocation=fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:17,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T12:21:17,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 5c909e1f401dd618f9fe9ecea4bdcfbc, server=fbad4c8a32e4,44527,1730981867254 in 173 msec 2024-11-07T12:21:17,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T12:21:17,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=5c909e1f401dd618f9fe9ecea4bdcfbc, ASSIGN in 328 msec 2024-11-07T12:21:17,077 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:21:17,077 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982077077"}]},"ts":"1730982077077"} 2024-11-07T12:21:17,078 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T12:21:17,081 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:21:17,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1650 sec 2024-11-07T12:21:17,743 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970048 expired clientIPAndPort=172.17.0.2:54398, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T12:21:18,922 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-07T12:21:18,922 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T12:21:18,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:21:18,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44527 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-07T12:21:18,926 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16774695992363581604 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T12:21:18,927 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T12:21:18,927 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T12:21:18,927 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T12:21:18,929 DEBUG [hconnection-0x195dddaf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:21:18,930 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39396, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:21:18,938 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T12:21:18,942 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T12:21:18,942 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T12:21:18,943 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x109e8270 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fbad4c8a32e4,44527,1730981867254, seqNum=-1] 2024-11-07T12:21:18,943 DEBUG [ReadOnlyZKClient-127.0.0.1:51230@0x109e8270 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:21:18,945 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39406, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:21:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:21:18,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:21:19,845 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970046 expired clientIPAndPort=172.17.0.2:54398, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:19,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-07T12:21:19,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-07T12:21:19,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:21:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T12:21:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2] 2024-11-07T12:21:20,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T12:21:21,427 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T12:21:21,942 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=979ms 2024-11-07T12:21:21,943 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=979ms 2024-11-07T12:21:21,943 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc., hostname=fbad4c8a32e4,44527,1730981867254, seqNum=2 from cache 2024-11-07T12:21:21,943 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T12:21:19.952Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-07T12:21:21.943Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=979ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=fbad4c8a32e4:44527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=980ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T12:21:21,944 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3002 2024-11-07T12:21:21,944 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bf05785 to 127.0.0.1:51230 2024-11-07T12:21:21,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:21,944 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T12:21:21,944 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:21:21,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:21,945 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x109e8270 to 127.0.0.1:51230 2024-11-07T12:21:21,953 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=230 (was 225) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 338), ProcessCount=11 (was 11), AvailableMemoryMB=2058 (was 1998) - AvailableMemoryMB LEAK? - 2024-11-07T12:21:21,954 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-07T12:21:21,954 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-07T12:21:21,954 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x551b3f6b to 127.0.0.1:51230 2024-11-07T12:21:21,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:21,954 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T12:21:21,954 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1426379418, stopped=false 2024-11-07T12:21:21,954 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=fbad4c8a32e4,36333,1730981865680 2024-11-07T12:21:21,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:21:21,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:21:21,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-07T12:21:21,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:21:21,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:21:21,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:21,957 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'fbad4c8a32e4,44527,1730981867254' ***** 2024-11-07T12:21:21,957 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-07T12:21:21,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:21:21,957 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T12:21:21,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:21:21,957 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 7db5f0ffc15ecc581283f92d21730cb9 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for b6063fc2dd91d0f4fc20bca211b18d17 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 96acfee49f6cde18af501f86ea60e0a4 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for bc6ca4f162ccd89027841a9df4cc6af5 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for bcdda0a06bb9bc89931546d07c0b88ed 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 0e1920f7313ec7776012f1bb237576bf 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 7dfe2b1bde36fc9612e43e4145496903 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 4ceec90215644686ca88a4a9060a8a10 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 6afeba0ade51e5d04366bab1ccc89b28 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 4bf92c6c7d15fd30800845193778cf48 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 66173e428a30393976f3b6c0f80d3be0 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for b472beaddbb1077158ef010b24cb5f9c 2024-11-07T12:21:21,958 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 6befa1d405285dfda10661543d4825a3 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 2c64a0e802a548485c4a2b86bcd23e46 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for bebca667db35bff8ba945d067d8e9101 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 5c909e1f401dd618f9fe9ecea4bdcfbc 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 69f29002b8ac38e61dd30e67f2cf97a7 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 40f79a6cfe590363e2b8a7ecd6208f96 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 46a4386907010562876d51566682636a 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 2afefd66dadc9c5d4483629ec062e7ed 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 57fa8c5121af12042d7adaedb84ee993 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 8bb4504b4e674ba204befe801bad627b 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 1545cc022e3b8dd3b28b3dd70139e99f 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:21:21,959 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7db5f0ffc15ecc581283f92d21730cb9, disabling compactions & flushes 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1224): stopping server fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:21,959 DEBUG [RS:0;fbad4c8a32e4:44527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:21,959 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:21:21,959 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T12:21:21,959 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. after waiting 0 ms 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T12:21:21,959 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-07T12:21:21,959 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-07T12:21:21,959 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:21:21,959 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7db5f0ffc15ecc581283f92d21730cb9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:21,959 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T12:21:21,960 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T12:21:21,960 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:21:21,960 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:21:21,960 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-07T12:21:21,960 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 1588230740, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7db5f0ffc15ecc581283f92d21730cb9, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, b6063fc2dd91d0f4fc20bca211b18d17, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:22,016 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/.tmp/testFamily/c892e42b4258416e9d33cb9b1e743232 is 61, key is row-0/testFamily:testQualifier/1730981907660/Put/seqid=0 2024-11-07T12:21:22,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741893_1069 (size=5049) 2024-11-07T12:21:22,029 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/.tmp/info/b87eb174d2a24216ae7d8672c2077bbe is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf./info:regioninfo/1730981903889/Put/seqid=0 2024-11-07T12:21:22,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741894_1070 (size=51873) 2024-11-07T12:21:22,160 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 1588230740, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7db5f0ffc15ecc581283f92d21730cb9, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, b6063fc2dd91d0f4fc20bca211b18d17, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:22,361 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 1588230740, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7db5f0ffc15ecc581283f92d21730cb9, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, b6063fc2dd91d0f4fc20bca211b18d17, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:22,426 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/.tmp/testFamily/c892e42b4258416e9d33cb9b1e743232 2024-11-07T12:21:22,435 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/.tmp/info/b87eb174d2a24216ae7d8672c2077bbe 2024-11-07T12:21:22,463 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T12:21:22,472 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/.tmp/testFamily/c892e42b4258416e9d33cb9b1e743232 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/testFamily/c892e42b4258416e9d33cb9b1e743232 2024-11-07T12:21:22,479 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/testFamily/c892e42b4258416e9d33cb9b1e743232, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:22,482 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7db5f0ffc15ecc581283f92d21730cb9 in 523ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:22,487 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7db5f0ffc15ecc581283f92d21730cb9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:22,490 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:21:22,490 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7db5f0ffc15ecc581283f92d21730cb9: 2024-11-07T12:21:22,490 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730981904635.7db5f0ffc15ecc581283f92d21730cb9. 2024-11-07T12:21:22,490 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b6063fc2dd91d0f4fc20bca211b18d17, disabling compactions & flushes 2024-11-07T12:21:22,490 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:21:22,491 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:21:22,491 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. after waiting 0 ms 2024-11-07T12:21:22,491 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:21:22,491 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b6063fc2dd91d0f4fc20bca211b18d17 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:22,491 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/.tmp/table/db59aaa2e97d4dc991d0c6009ad6b1b1 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1730981903902/Put/seqid=0 2024-11-07T12:21:22,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741895_1071 (size=9821) 2024-11-07T12:21:22,508 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/.tmp/testFamily/3d5c9c5d294f4b2e8ece7dedf05323bd is 61, key is row-0/testFamily:testQualifier/1730981927892/Put/seqid=0 2024-11-07T12:21:22,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741896_1072 (size=5049) 2024-11-07T12:21:22,561 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 1588230740, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, b6063fc2dd91d0f4fc20bca211b18d17, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:22,597 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:21:22,761 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 1588230740, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, b6063fc2dd91d0f4fc20bca211b18d17, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:22,897 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/.tmp/table/db59aaa2e97d4dc991d0c6009ad6b1b1 2024-11-07T12:21:22,905 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/.tmp/info/b87eb174d2a24216ae7d8672c2077bbe as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/info/b87eb174d2a24216ae7d8672c2077bbe 2024-11-07T12:21:22,911 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/info/b87eb174d2a24216ae7d8672c2077bbe, entries=290, sequenceid=149, filesize=50.7 K 2024-11-07T12:21:22,913 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/.tmp/table/db59aaa2e97d4dc991d0c6009ad6b1b1 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/table/db59aaa2e97d4dc991d0c6009ad6b1b1 2024-11-07T12:21:22,913 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/.tmp/testFamily/3d5c9c5d294f4b2e8ece7dedf05323bd 2024-11-07T12:21:22,917 INFO [regionserver/fbad4c8a32e4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T12:21:22,917 INFO [regionserver/fbad4c8a32e4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T12:21:22,920 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/.tmp/testFamily/3d5c9c5d294f4b2e8ece7dedf05323bd as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/testFamily/3d5c9c5d294f4b2e8ece7dedf05323bd 2024-11-07T12:21:22,920 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/table/db59aaa2e97d4dc991d0c6009ad6b1b1, entries=58, sequenceid=149, filesize=9.6 K 2024-11-07T12:21:22,921 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 961ms, sequenceid=149, compaction requested=false 2024-11-07T12:21:22,925 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-07T12:21:22,926 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:21:22,926 INFO [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T12:21:22,926 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T12:21:22,926 DEBUG [RS_CLOSE_META-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T12:21:22,927 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/testFamily/3d5c9c5d294f4b2e8ece7dedf05323bd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:22,928 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b6063fc2dd91d0f4fc20bca211b18d17 in 437ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:22,933 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_0_-false/b6063fc2dd91d0f4fc20bca211b18d17/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:22,934 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:21:22,934 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b6063fc2dd91d0f4fc20bca211b18d17: 2024-11-07T12:21:22,934 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730981926850.b6063fc2dd91d0f4fc20bca211b18d17. 2024-11-07T12:21:22,934 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 96acfee49f6cde18af501f86ea60e0a4, disabling compactions & flushes 2024-11-07T12:21:22,934 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:21:22,934 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:21:22,934 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. after waiting 0 ms 2024-11-07T12:21:22,934 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:21:22,934 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 96acfee49f6cde18af501f86ea60e0a4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:22,954 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/.tmp/testFamily/2c069e92d4d14cf4a5aab5477ad86cb0 is 61, key is row-0/testFamily:testQualifier/1730982038632/Put/seqid=0 2024-11-07T12:21:22,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741897_1073 (size=5049) 2024-11-07T12:21:22,961 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-07T12:21:22,962 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:23,162 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:23,359 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/.tmp/testFamily/2c069e92d4d14cf4a5aab5477ad86cb0 2024-11-07T12:21:23,362 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, 96acfee49f6cde18af501f86ea60e0a4, b472beaddbb1077158ef010b24cb5f9c, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:23,368 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/.tmp/testFamily/2c069e92d4d14cf4a5aab5477ad86cb0 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/testFamily/2c069e92d4d14cf4a5aab5477ad86cb0 2024-11-07T12:21:23,374 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/testFamily/2c069e92d4d14cf4a5aab5477ad86cb0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:23,377 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 96acfee49f6cde18af501f86ea60e0a4 in 443ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:23,381 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_1_-true/96acfee49f6cde18af501f86ea60e0a4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:23,382 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:21:23,382 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 96acfee49f6cde18af501f86ea60e0a4: 2024-11-07T12:21:23,382 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1730982035612.96acfee49f6cde18af501f86ea60e0a4. 2024-11-07T12:21:23,382 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bc6ca4f162ccd89027841a9df4cc6af5, disabling compactions & flushes 2024-11-07T12:21:23,382 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:23,382 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:23,382 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. after waiting 0 ms 2024-11-07T12:21:23,382 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:23,382 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bc6ca4f162ccd89027841a9df4cc6af5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:23,398 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/.tmp/testFamily/2ca6e738cc35468f82a74eb35f70c879 is 61, key is row-0/testFamily:testQualifier/1730982066869/Put/seqid=0 2024-11-07T12:21:23,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741898_1074 (size=5049) 2024-11-07T12:21:23,562 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:23,763 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, bc6ca4f162ccd89027841a9df4cc6af5, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:23,804 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/.tmp/testFamily/2ca6e738cc35468f82a74eb35f70c879 2024-11-07T12:21:23,813 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/.tmp/testFamily/2ca6e738cc35468f82a74eb35f70c879 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/testFamily/2ca6e738cc35468f82a74eb35f70c879 2024-11-07T12:21:23,821 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/testFamily/2ca6e738cc35468f82a74eb35f70c879, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:23,822 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bc6ca4f162ccd89027841a9df4cc6af5 in 440ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:23,827 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_1_-true/bc6ca4f162ccd89027841a9df4cc6af5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:23,828 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:23,828 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bc6ca4f162ccd89027841a9df4cc6af5: 2024-11-07T12:21:23,828 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1730982063851.bc6ca4f162ccd89027841a9df4cc6af5. 2024-11-07T12:21:23,828 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bcdda0a06bb9bc89931546d07c0b88ed, disabling compactions & flushes 2024-11-07T12:21:23,828 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:21:23,828 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:21:23,828 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. after waiting 0 ms 2024-11-07T12:21:23,828 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:21:23,828 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bcdda0a06bb9bc89931546d07c0b88ed 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:23,845 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/.tmp/testFamily/2745a31b20064c09bf7121fe1cbb8a0d is 61, key is row-0/testFamily:testQualifier/1730982020403/Put/seqid=0 2024-11-07T12:21:23,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741899_1075 (size=5049) 2024-11-07T12:21:23,949 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1672048081345970041 expired clientIPAndPort=172.17.0.2:40292, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:21:23,963 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-07T12:21:23,963 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:24,163 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, bcdda0a06bb9bc89931546d07c0b88ed, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:24,252 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/.tmp/testFamily/2745a31b20064c09bf7121fe1cbb8a0d 2024-11-07T12:21:24,260 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/.tmp/testFamily/2745a31b20064c09bf7121fe1cbb8a0d as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/testFamily/2745a31b20064c09bf7121fe1cbb8a0d 2024-11-07T12:21:24,267 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/testFamily/2745a31b20064c09bf7121fe1cbb8a0d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:24,267 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bcdda0a06bb9bc89931546d07c0b88ed in 439ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:24,272 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_1_-true/bcdda0a06bb9bc89931546d07c0b88ed/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:24,273 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:21:24,273 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bcdda0a06bb9bc89931546d07c0b88ed: 2024-11-07T12:21:24,273 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1730982019379.bcdda0a06bb9bc89931546d07c0b88ed. 2024-11-07T12:21:24,273 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0e1920f7313ec7776012f1bb237576bf, disabling compactions & flushes 2024-11-07T12:21:24,273 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:21:24,273 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:21:24,273 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. after waiting 0 ms 2024-11-07T12:21:24,273 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:21:24,273 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0e1920f7313ec7776012f1bb237576bf 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:24,290 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/.tmp/testFamily/625f9633909c4c59a436b5e09594f2cf is 61, key is row-0/testFamily:testQualifier/1730981904475/Put/seqid=0 2024-11-07T12:21:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741900_1076 (size=5049) 2024-11-07T12:21:24,364 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:24,564 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 0e1920f7313ec7776012f1bb237576bf, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:24,696 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/.tmp/testFamily/625f9633909c4c59a436b5e09594f2cf 2024-11-07T12:21:24,703 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/.tmp/testFamily/625f9633909c4c59a436b5e09594f2cf as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/testFamily/625f9633909c4c59a436b5e09594f2cf 2024-11-07T12:21:24,709 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/testFamily/625f9633909c4c59a436b5e09594f2cf, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:24,710 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0e1920f7313ec7776012f1bb237576bf in 437ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:24,714 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/0e1920f7313ec7776012f1bb237576bf/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:24,715 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:21:24,715 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0e1920f7313ec7776012f1bb237576bf: 2024-11-07T12:21:24,715 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730981903428.0e1920f7313ec7776012f1bb237576bf. 2024-11-07T12:21:24,715 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7dfe2b1bde36fc9612e43e4145496903, disabling compactions & flushes 2024-11-07T12:21:24,715 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:24,715 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:24,715 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. after waiting 0 ms 2024-11-07T12:21:24,715 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:24,715 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7dfe2b1bde36fc9612e43e4145496903 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:24,731 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/.tmp/testFamily/cee40b5d67e34cf6aa86ef8188256ce8 is 61, key is row-0/testFamily:testQualifier/1730982011992/Put/seqid=0 2024-11-07T12:21:24,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741901_1077 (size=5049) 2024-11-07T12:21:24,764 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:24,964 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-07T12:21:24,964 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 7dfe2b1bde36fc9612e43e4145496903, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:25,135 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/.tmp/testFamily/cee40b5d67e34cf6aa86ef8188256ce8 2024-11-07T12:21:25,144 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/.tmp/testFamily/cee40b5d67e34cf6aa86ef8188256ce8 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/testFamily/cee40b5d67e34cf6aa86ef8188256ce8 2024-11-07T12:21:25,150 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/testFamily/cee40b5d67e34cf6aa86ef8188256ce8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:25,151 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7dfe2b1bde36fc9612e43e4145496903 in 436ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:25,155 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextException_1_-true/7dfe2b1bde36fc9612e43e4145496903/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:25,156 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:25,156 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7dfe2b1bde36fc9612e43e4145496903: 2024-11-07T12:21:25,157 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1730982010971.7dfe2b1bde36fc9612e43e4145496903. 2024-11-07T12:21:25,157 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4ceec90215644686ca88a4a9060a8a10, disabling compactions & flushes 2024-11-07T12:21:25,157 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:21:25,157 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:21:25,157 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. after waiting 0 ms 2024-11-07T12:21:25,157 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:21:25,157 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4ceec90215644686ca88a4a9060a8a10 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:25,165 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:25,174 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/.tmp/testFamily/0cfe773abd2c4f6bbf853a0d20def966 is 61, key is row-0/testFamily:testQualifier/1730981986454/Put/seqid=0 2024-11-07T12:21:25,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741902_1078 (size=5049) 2024-11-07T12:21:25,365 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:25,565 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 4ceec90215644686ca88a4a9060a8a10, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:25,579 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/.tmp/testFamily/0cfe773abd2c4f6bbf853a0d20def966 2024-11-07T12:21:25,588 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/.tmp/testFamily/0cfe773abd2c4f6bbf853a0d20def966 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/testFamily/0cfe773abd2c4f6bbf853a0d20def966 2024-11-07T12:21:25,595 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/testFamily/0cfe773abd2c4f6bbf853a0d20def966, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:25,596 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4ceec90215644686ca88a4a9060a8a10 in 439ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:25,600 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/4ceec90215644686ca88a4a9060a8a10/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:25,601 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:21:25,601 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4ceec90215644686ca88a4a9060a8a10: 2024-11-07T12:21:25,601 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730981983426.4ceec90215644686ca88a4a9060a8a10. 2024-11-07T12:21:25,601 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6afeba0ade51e5d04366bab1ccc89b28, disabling compactions & flushes 2024-11-07T12:21:25,601 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:21:25,601 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:21:25,601 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. after waiting 0 ms 2024-11-07T12:21:25,601 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:21:25,601 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6afeba0ade51e5d04366bab1ccc89b28 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:25,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/.tmp/testFamily/8725caae168d4d33afb6105338f99443 is 61, key is row-0/testFamily:testQualifier/1730981990552/Put/seqid=0 2024-11-07T12:21:25,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741903_1079 (size=5049) 2024-11-07T12:21:25,766 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:25,966 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-07T12:21:25,966 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6afeba0ade51e5d04366bab1ccc89b28, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:26,024 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/.tmp/testFamily/8725caae168d4d33afb6105338f99443 2024-11-07T12:21:26,031 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/.tmp/testFamily/8725caae168d4d33afb6105338f99443 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/testFamily/8725caae168d4d33afb6105338f99443 2024-11-07T12:21:26,038 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/testFamily/8725caae168d4d33afb6105338f99443, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:26,039 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6afeba0ade51e5d04366bab1ccc89b28 in 438ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:26,044 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/6afeba0ade51e5d04366bab1ccc89b28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:26,044 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:21:26,045 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6afeba0ade51e5d04366bab1ccc89b28: 2024-11-07T12:21:26,045 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730981989527.6afeba0ade51e5d04366bab1ccc89b28. 2024-11-07T12:21:26,045 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing db5597a6c7aaebb9f2b1975a84ab1323, disabling compactions & flushes 2024-11-07T12:21:26,045 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:21:26,045 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:21:26,045 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. after waiting 0 ms 2024-11-07T12:21:26,045 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:21:26,045 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing db5597a6c7aaebb9f2b1975a84ab1323 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:26,066 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/.tmp/testFamily/3398b83620204042bbc96eafde7537f9 is 61, key is row-0/testFamily:testQualifier/1730982013094/Put/seqid=0 2024-11-07T12:21:26,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741904_1080 (size=5049) 2024-11-07T12:21:26,166 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:26,366 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb, db5597a6c7aaebb9f2b1975a84ab1323 2024-11-07T12:21:26,473 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/.tmp/testFamily/3398b83620204042bbc96eafde7537f9 2024-11-07T12:21:26,481 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/.tmp/testFamily/3398b83620204042bbc96eafde7537f9 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/testFamily/3398b83620204042bbc96eafde7537f9 2024-11-07T12:21:26,488 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/testFamily/3398b83620204042bbc96eafde7537f9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:26,489 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for db5597a6c7aaebb9f2b1975a84ab1323 in 444ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:26,493 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_1_-true/db5597a6c7aaebb9f2b1975a84ab1323/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:26,494 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:21:26,494 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for db5597a6c7aaebb9f2b1975a84ab1323: 2024-11-07T12:21:26,494 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1730982012074.db5597a6c7aaebb9f2b1975a84ab1323. 2024-11-07T12:21:26,494 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4bf92c6c7d15fd30800845193778cf48, disabling compactions & flushes 2024-11-07T12:21:26,494 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:21:26,494 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:21:26,494 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. after waiting 0 ms 2024-11-07T12:21:26,494 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:21:26,494 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4bf92c6c7d15fd30800845193778cf48 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:26,513 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/.tmp/testFamily/89f35986f1874386ba3edfa64f165015 is 61, key is row-0/testFamily:testQualifier/1730982034072/Put/seqid=0 2024-11-07T12:21:26,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741905_1081 (size=5049) 2024-11-07T12:21:26,567 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:26,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T12:21:26,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T12:21:26,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T12:21:26,581 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T12:21:26,767 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 4bf92c6c7d15fd30800845193778cf48, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:26,918 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/.tmp/testFamily/89f35986f1874386ba3edfa64f165015 2024-11-07T12:21:26,926 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/.tmp/testFamily/89f35986f1874386ba3edfa64f165015 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/testFamily/89f35986f1874386ba3edfa64f165015 2024-11-07T12:21:26,931 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/testFamily/89f35986f1874386ba3edfa64f165015, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:26,932 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4bf92c6c7d15fd30800845193778cf48 in 438ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:26,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_1_-true/4bf92c6c7d15fd30800845193778cf48/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:26,937 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:21:26,937 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4bf92c6c7d15fd30800845193778cf48: 2024-11-07T12:21:26,937 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1730982031053.4bf92c6c7d15fd30800845193778cf48. 2024-11-07T12:21:26,937 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 66173e428a30393976f3b6c0f80d3be0, disabling compactions & flushes 2024-11-07T12:21:26,937 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:21:26,937 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:21:26,937 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. after waiting 0 ms 2024-11-07T12:21:26,937 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:21:26,937 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 66173e428a30393976f3b6c0f80d3be0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:26,954 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/.tmp/testFamily/7412c2afcab044e8aedf7af380824bb5 is 61, key is row-0/testFamily:testQualifier/1730982044697/Put/seqid=0 2024-11-07T12:21:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741906_1082 (size=5049) 2024-11-07T12:21:26,967 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 18 regions to close 2024-11-07T12:21:26,967 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:27,168 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:27,359 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/.tmp/testFamily/7412c2afcab044e8aedf7af380824bb5 2024-11-07T12:21:27,366 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/.tmp/testFamily/7412c2afcab044e8aedf7af380824bb5 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/testFamily/7412c2afcab044e8aedf7af380824bb5 2024-11-07T12:21:27,368 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 66173e428a30393976f3b6c0f80d3be0, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:27,372 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/testFamily/7412c2afcab044e8aedf7af380824bb5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:27,373 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66173e428a30393976f3b6c0f80d3be0 in 436ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:27,377 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/66173e428a30393976f3b6c0f80d3be0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:27,378 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:21:27,378 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 66173e428a30393976f3b6c0f80d3be0: 2024-11-07T12:21:27,378 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1730982041676.66173e428a30393976f3b6c0f80d3be0. 2024-11-07T12:21:27,378 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b472beaddbb1077158ef010b24cb5f9c, disabling compactions & flushes 2024-11-07T12:21:27,378 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:21:27,378 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:21:27,378 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. after waiting 0 ms 2024-11-07T12:21:27,378 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:21:27,378 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b472beaddbb1077158ef010b24cb5f9c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:27,394 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/.tmp/testFamily/01187f27ed8e48b0be1e7f7fe9727f01 is 61, key is row-0/testFamily:testQualifier/1730981938317/Put/seqid=0 2024-11-07T12:21:27,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741907_1083 (size=5049) 2024-11-07T12:21:27,568 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:27,768 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, b472beaddbb1077158ef010b24cb5f9c, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:27,799 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/.tmp/testFamily/01187f27ed8e48b0be1e7f7fe9727f01 2024-11-07T12:21:27,807 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/.tmp/testFamily/01187f27ed8e48b0be1e7f7fe9727f01 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/testFamily/01187f27ed8e48b0be1e7f7fe9727f01 2024-11-07T12:21:27,813 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/testFamily/01187f27ed8e48b0be1e7f7fe9727f01, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:27,814 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b472beaddbb1077158ef010b24cb5f9c in 436ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:27,818 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNextAsync_0_-false/b472beaddbb1077158ef010b24cb5f9c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:27,819 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:21:27,819 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b472beaddbb1077158ef010b24cb5f9c: 2024-11-07T12:21:27,819 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730981937297.b472beaddbb1077158ef010b24cb5f9c. 2024-11-07T12:21:27,819 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6befa1d405285dfda10661543d4825a3, disabling compactions & flushes 2024-11-07T12:21:27,819 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:21:27,819 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:21:27,819 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. after waiting 0 ms 2024-11-07T12:21:27,819 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:21:27,819 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6befa1d405285dfda10661543d4825a3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:27,835 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/.tmp/testFamily/f0c428557e664d0faa824919e069f728 is 61, key is row-0/testFamily:testQualifier/1730981934247/Put/seqid=0 2024-11-07T12:21:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741908_1084 (size=5049) 2024-11-07T12:21:27,969 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-07T12:21:27,969 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:27,991 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:21:28,169 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 6befa1d405285dfda10661543d4825a3, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101, c2ca8fcef0a22d2abc17945041033cbb 2024-11-07T12:21:28,241 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/.tmp/testFamily/f0c428557e664d0faa824919e069f728 2024-11-07T12:21:28,248 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/.tmp/testFamily/f0c428557e664d0faa824919e069f728 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/testFamily/f0c428557e664d0faa824919e069f728 2024-11-07T12:21:28,253 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/testFamily/f0c428557e664d0faa824919e069f728, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:28,254 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6befa1d405285dfda10661543d4825a3 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:28,258 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/6befa1d405285dfda10661543d4825a3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:28,259 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:21:28,259 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6befa1d405285dfda10661543d4825a3: 2024-11-07T12:21:28,259 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730981931225.6befa1d405285dfda10661543d4825a3. 2024-11-07T12:21:28,259 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c2ca8fcef0a22d2abc17945041033cbb, disabling compactions & flushes 2024-11-07T12:21:28,259 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:21:28,259 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:21:28,259 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. after waiting 0 ms 2024-11-07T12:21:28,259 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:21:28,259 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c2ca8fcef0a22d2abc17945041033cbb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:28,275 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/.tmp/testFamily/2e1ab28168b14c4a8e5064c3c3fb2012 is 61, key is row-0/testFamily:testQualifier/1730981942397/Put/seqid=0 2024-11-07T12:21:28,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741909_1085 (size=5049) 2024-11-07T12:21:28,280 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/.tmp/testFamily/2e1ab28168b14c4a8e5064c3c3fb2012 2024-11-07T12:21:28,288 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/.tmp/testFamily/2e1ab28168b14c4a8e5064c3c3fb2012 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/testFamily/2e1ab28168b14c4a8e5064c3c3fb2012 2024-11-07T12:21:28,294 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/testFamily/2e1ab28168b14c4a8e5064c3c3fb2012, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:28,294 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c2ca8fcef0a22d2abc17945041033cbb in 35ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:28,298 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_0_-false/c2ca8fcef0a22d2abc17945041033cbb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:28,299 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:21:28,299 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c2ca8fcef0a22d2abc17945041033cbb: 2024-11-07T12:21:28,299 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730981941375.c2ca8fcef0a22d2abc17945041033cbb. 2024-11-07T12:21:28,299 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2c64a0e802a548485c4a2b86bcd23e46, disabling compactions & flushes 2024-11-07T12:21:28,299 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:21:28,299 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:21:28,299 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. after waiting 0 ms 2024-11-07T12:21:28,299 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:21:28,299 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2c64a0e802a548485c4a2b86bcd23e46 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:28,315 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/.tmp/testFamily/a39ff5d17561472e9c648b9d57986731 is 61, key is row-0/testFamily:testQualifier/1730981929106/Put/seqid=0 2024-11-07T12:21:28,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741910_1086 (size=5049) 2024-11-07T12:21:28,369 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101 2024-11-07T12:21:28,569 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 2c64a0e802a548485c4a2b86bcd23e46, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101 2024-11-07T12:21:28,721 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/.tmp/testFamily/a39ff5d17561472e9c648b9d57986731 2024-11-07T12:21:28,728 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/.tmp/testFamily/a39ff5d17561472e9c648b9d57986731 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/testFamily/a39ff5d17561472e9c648b9d57986731 2024-11-07T12:21:28,733 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/testFamily/a39ff5d17561472e9c648b9d57986731, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:28,734 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2c64a0e802a548485c4a2b86bcd23e46 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:28,738 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnRenewLease_0_-false/2c64a0e802a548485c4a2b86bcd23e46/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:28,739 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:21:28,739 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2c64a0e802a548485c4a2b86bcd23e46: 2024-11-07T12:21:28,739 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730981928028.2c64a0e802a548485c4a2b86bcd23e46. 2024-11-07T12:21:28,739 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bebca667db35bff8ba945d067d8e9101, disabling compactions & flushes 2024-11-07T12:21:28,739 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:21:28,739 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:21:28,739 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. after waiting 0 ms 2024-11-07T12:21:28,739 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:21:28,739 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bebca667db35bff8ba945d067d8e9101 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:28,755 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/.tmp/testFamily/cd2c7622d29d4036bcace643b3a4475f is 61, key is row-0/testFamily:testQualifier/1730981991768/Put/seqid=0 2024-11-07T12:21:28,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741911_1087 (size=5049) 2024-11-07T12:21:28,770 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101 2024-11-07T12:21:28,970 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 13 regions to close 2024-11-07T12:21:28,970 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101 2024-11-07T12:21:29,160 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/.tmp/testFamily/cd2c7622d29d4036bcace643b3a4475f 2024-11-07T12:21:29,168 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/.tmp/testFamily/cd2c7622d29d4036bcace643b3a4475f as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/testFamily/cd2c7622d29d4036bcace643b3a4475f 2024-11-07T12:21:29,170 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d, bebca667db35bff8ba945d067d8e9101 2024-11-07T12:21:29,173 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/testFamily/cd2c7622d29d4036bcace643b3a4475f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:29,174 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bebca667db35bff8ba945d067d8e9101 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:29,177 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/bebca667db35bff8ba945d067d8e9101/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:29,178 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:21:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bebca667db35bff8ba945d067d8e9101: 2024-11-07T12:21:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730981990715.bebca667db35bff8ba945d067d8e9101. 2024-11-07T12:21:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5c909e1f401dd618f9fe9ecea4bdcfbc, disabling compactions & flushes 2024-11-07T12:21:29,178 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. after waiting 0 ms 2024-11-07T12:21:29,178 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:29,178 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 5c909e1f401dd618f9fe9ecea4bdcfbc 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:29,194 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/.tmp/testFamily/49a73fc9f8a945a39d44cc486942c495 is 61, key is row-0/testFamily:testQualifier/1730982078932/Put/seqid=0 2024-11-07T12:21:29,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741912_1088 (size=5049) 2024-11-07T12:21:29,370 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:29,571 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 5c909e1f401dd618f9fe9ecea4bdcfbc, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:29,599 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/.tmp/testFamily/49a73fc9f8a945a39d44cc486942c495 2024-11-07T12:21:29,606 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/.tmp/testFamily/49a73fc9f8a945a39d44cc486942c495 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/testFamily/49a73fc9f8a945a39d44cc486942c495 2024-11-07T12:21:29,612 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/testFamily/49a73fc9f8a945a39d44cc486942c495, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:29,613 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5c909e1f401dd618f9fe9ecea4bdcfbc in 435ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:29,617 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/5c909e1f401dd618f9fe9ecea4bdcfbc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:29,618 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:29,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5c909e1f401dd618f9fe9ecea4bdcfbc: 2024-11-07T12:21:29,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1730982075915.5c909e1f401dd618f9fe9ecea4bdcfbc. 2024-11-07T12:21:29,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 69f29002b8ac38e61dd30e67f2cf97a7, disabling compactions & flushes 2024-11-07T12:21:29,618 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:21:29,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:21:29,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. after waiting 0 ms 2024-11-07T12:21:29,618 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:21:29,618 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 69f29002b8ac38e61dd30e67f2cf97a7 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-07T12:21:29,634 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/.tmp/info/6b06aa4763194ce08be199739125d34e is 45, key is default/info:d/1730981903038/Put/seqid=0 2024-11-07T12:21:29,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741913_1089 (size=5037) 2024-11-07T12:21:29,771 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:29,971 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 11 regions to close 2024-11-07T12:21:29,971 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 69f29002b8ac38e61dd30e67f2cf97a7, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:30,040 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/.tmp/info/6b06aa4763194ce08be199739125d34e 2024-11-07T12:21:30,047 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/.tmp/info/6b06aa4763194ce08be199739125d34e as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/info/6b06aa4763194ce08be199739125d34e 2024-11-07T12:21:30,052 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/info/6b06aa4763194ce08be199739125d34e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T12:21:30,053 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 69f29002b8ac38e61dd30e67f2cf97a7 in 435ms, sequenceid=6, compaction requested=false 2024-11-07T12:21:30,057 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/namespace/69f29002b8ac38e61dd30e67f2cf97a7/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T12:21:30,058 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:21:30,058 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 69f29002b8ac38e61dd30e67f2cf97a7: 2024-11-07T12:21:30,058 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1730981902224.69f29002b8ac38e61dd30e67f2cf97a7. 2024-11-07T12:21:30,058 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 40f79a6cfe590363e2b8a7ecd6208f96, disabling compactions & flushes 2024-11-07T12:21:30,058 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:21:30,058 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:21:30,058 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. after waiting 0 ms 2024-11-07T12:21:30,058 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:21:30,058 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 40f79a6cfe590363e2b8a7ecd6208f96 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:30,073 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/.tmp/testFamily/2cdf5b5cc087493c8ecdf50d742b4464 is 61, key is row-0/testFamily:testQualifier/1730981977284/Put/seqid=0 2024-11-07T12:21:30,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741914_1090 (size=5049) 2024-11-07T12:21:30,171 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:30,372 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 40f79a6cfe590363e2b8a7ecd6208f96, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:30,479 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/.tmp/testFamily/2cdf5b5cc087493c8ecdf50d742b4464 2024-11-07T12:21:30,485 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/.tmp/testFamily/2cdf5b5cc087493c8ecdf50d742b4464 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/testFamily/2cdf5b5cc087493c8ecdf50d742b4464 2024-11-07T12:21:30,491 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/testFamily/2cdf5b5cc087493c8ecdf50d742b4464, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:30,492 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 40f79a6cfe590363e2b8a7ecd6208f96 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:30,496 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNext_0_-false/40f79a6cfe590363e2b8a7ecd6208f96/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:30,496 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:21:30,496 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 40f79a6cfe590363e2b8a7ecd6208f96: 2024-11-07T12:21:30,496 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730981974259.40f79a6cfe590363e2b8a7ecd6208f96. 2024-11-07T12:21:30,497 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 46a4386907010562876d51566682636a, disabling compactions & flushes 2024-11-07T12:21:30,497 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:21:30,497 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:21:30,497 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. after waiting 0 ms 2024-11-07T12:21:30,497 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:21:30,497 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 46a4386907010562876d51566682636a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:30,512 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/.tmp/testFamily/c420fcb020064ae49de8e95d4ff2115c is 61, key is row-0/testFamily:testQualifier/1730981950021/Put/seqid=0 2024-11-07T12:21:30,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741915_1091 (size=5049) 2024-11-07T12:21:30,572 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:30,772 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 46a4386907010562876d51566682636a, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:30,917 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/.tmp/testFamily/c420fcb020064ae49de8e95d4ff2115c 2024-11-07T12:21:30,924 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/.tmp/testFamily/c420fcb020064ae49de8e95d4ff2115c as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/testFamily/c420fcb020064ae49de8e95d4ff2115c 2024-11-07T12:21:30,930 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/testFamily/c420fcb020064ae49de8e95d4ff2115c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:30,931 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46a4386907010562876d51566682636a in 434ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:30,935 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnClose_0_-false/46a4386907010562876d51566682636a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:30,936 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:21:30,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 46a4386907010562876d51566682636a: 2024-11-07T12:21:30,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1730981948972.46a4386907010562876d51566682636a. 2024-11-07T12:21:30,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2afefd66dadc9c5d4483629ec062e7ed, disabling compactions & flushes 2024-11-07T12:21:30,936 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:21:30,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:21:30,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. after waiting 0 ms 2024-11-07T12:21:30,936 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:21:30,936 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2afefd66dadc9c5d4483629ec062e7ed 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:30,952 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/.tmp/testFamily/1f1431b831764ec79359852ec8db8dd7 is 61, key is row-0/testFamily:testQualifier/1730981917768/Put/seqid=0 2024-11-07T12:21:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741916_1092 (size=5049) 2024-11-07T12:21:30,973 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 8 regions to close 2024-11-07T12:21:30,973 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1603): Online Regions={2afefd66dadc9c5d4483629ec062e7ed=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed., 57fa8c5121af12042d7adaedb84ee993=testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993., 8bb4504b4e674ba204befe801bad627b=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., 1545cc022e3b8dd3b28b3dd70139e99f=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., be85db67bdb4b8cbb0470630805ebc5d=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d., 6497436b1160bd1dd303db7791f75ae1=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., 15982a8bd19de3fe41d3798072bfe2a3=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3., 0b0d536521b4033814c4e766b23cf3ae=hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.} 2024-11-07T12:21:30,973 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:31,173 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:31,357 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/.tmp/testFamily/1f1431b831764ec79359852ec8db8dd7 2024-11-07T12:21:31,364 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/.tmp/testFamily/1f1431b831764ec79359852ec8db8dd7 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/testFamily/1f1431b831764ec79359852ec8db8dd7 2024-11-07T12:21:31,371 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/testFamily/1f1431b831764ec79359852ec8db8dd7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:31,371 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2afefd66dadc9c5d4483629ec062e7ed in 435ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:31,373 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 2afefd66dadc9c5d4483629ec062e7ed, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:31,375 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/2afefd66dadc9c5d4483629ec062e7ed/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:31,376 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:21:31,376 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2afefd66dadc9c5d4483629ec062e7ed: 2024-11-07T12:21:31,376 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730981916732.2afefd66dadc9c5d4483629ec062e7ed. 2024-11-07T12:21:31,376 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 57fa8c5121af12042d7adaedb84ee993, disabling compactions & flushes 2024-11-07T12:21:31,376 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:21:31,376 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:21:31,376 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. after waiting 0 ms 2024-11-07T12:21:31,376 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:21:31,376 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 57fa8c5121af12042d7adaedb84ee993 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:31,392 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/.tmp/testFamily/bb755d444e2d4454b5d088492a0e5ca6 is 61, key is row-0/testFamily:testQualifier/1730981952660/Put/seqid=0 2024-11-07T12:21:31,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741917_1093 (size=5049) 2024-11-07T12:21:31,573 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:31,774 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 57fa8c5121af12042d7adaedb84ee993, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:31,797 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/.tmp/testFamily/bb755d444e2d4454b5d088492a0e5ca6 2024-11-07T12:21:31,805 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/.tmp/testFamily/bb755d444e2d4454b5d088492a0e5ca6 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/testFamily/bb755d444e2d4454b5d088492a0e5ca6 2024-11-07T12:21:31,810 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/testFamily/bb755d444e2d4454b5d088492a0e5ca6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:31,811 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 57fa8c5121af12042d7adaedb84ee993 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:31,816 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnNext_0_-false/57fa8c5121af12042d7adaedb84ee993/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:31,817 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:21:31,817 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 57fa8c5121af12042d7adaedb84ee993: 2024-11-07T12:21:31,817 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1730981951598.57fa8c5121af12042d7adaedb84ee993. 2024-11-07T12:21:31,817 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8bb4504b4e674ba204befe801bad627b, disabling compactions & flushes 2024-11-07T12:21:31,817 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:31,817 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:31,817 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. after waiting 0 ms 2024-11-07T12:21:31,817 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:31,817 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8bb4504b4e674ba204befe801bad627b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:31,833 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/.tmp/testFamily/6924223ff8034e8da3979a8df5229cc4 is 61, key is row-0/testFamily:testQualifier/1730982016189/Put/seqid=0 2024-11-07T12:21:31,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741918_1094 (size=5049) 2024-11-07T12:21:31,974 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 6 regions to close 2024-11-07T12:21:31,974 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1603): Online Regions={8bb4504b4e674ba204befe801bad627b=testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b., 1545cc022e3b8dd3b28b3dd70139e99f=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f., be85db67bdb4b8cbb0470630805ebc5d=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d., 6497436b1160bd1dd303db7791f75ae1=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., 15982a8bd19de3fe41d3798072bfe2a3=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3., 0b0d536521b4033814c4e766b23cf3ae=hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.} 2024-11-07T12:21:31,974 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:32,174 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1, 8bb4504b4e674ba204befe801bad627b, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:32,239 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/.tmp/testFamily/6924223ff8034e8da3979a8df5229cc4 2024-11-07T12:21:32,245 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/.tmp/testFamily/6924223ff8034e8da3979a8df5229cc4 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/testFamily/6924223ff8034e8da3979a8df5229cc4 2024-11-07T12:21:32,251 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/testFamily/6924223ff8034e8da3979a8df5229cc4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:32,252 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8bb4504b4e674ba204befe801bad627b in 434ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:32,255 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8bb4504b4e674ba204befe801bad627b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:32,256 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:32,256 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8bb4504b4e674ba204befe801bad627b: 2024-11-07T12:21:32,256 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1730982015167.8bb4504b4e674ba204befe801bad627b. 2024-11-07T12:21:32,256 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1545cc022e3b8dd3b28b3dd70139e99f, disabling compactions & flushes 2024-11-07T12:21:32,256 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:21:32,256 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:21:32,256 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. after waiting 0 ms 2024-11-07T12:21:32,256 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:21:32,256 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1545cc022e3b8dd3b28b3dd70139e99f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:32,271 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/.tmp/testFamily/d247db3bc6e4479687b6f0fbe050365f is 61, key is row-0/testFamily:testQualifier/1730981965148/Put/seqid=0 2024-11-07T12:21:32,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741919_1095 (size=5049) 2024-11-07T12:21:32,375 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:32,575 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 1545cc022e3b8dd3b28b3dd70139e99f, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:32,676 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/.tmp/testFamily/d247db3bc6e4479687b6f0fbe050365f 2024-11-07T12:21:32,682 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/.tmp/testFamily/d247db3bc6e4479687b6f0fbe050365f as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/testFamily/d247db3bc6e4479687b6f0fbe050365f 2024-11-07T12:21:32,688 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/testFamily/d247db3bc6e4479687b6f0fbe050365f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:32,689 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1545cc022e3b8dd3b28b3dd70139e99f in 433ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:32,693 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1545cc022e3b8dd3b28b3dd70139e99f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:32,693 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:21:32,693 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1545cc022e3b8dd3b28b3dd70139e99f: 2024-11-07T12:21:32,693 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730981964095.1545cc022e3b8dd3b28b3dd70139e99f. 2024-11-07T12:21:32,694 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing be85db67bdb4b8cbb0470630805ebc5d, disabling compactions & flushes 2024-11-07T12:21:32,694 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:21:32,694 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:21:32,694 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. after waiting 0 ms 2024-11-07T12:21:32,694 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:21:32,694 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing be85db67bdb4b8cbb0470630805ebc5d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:32,709 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/.tmp/testFamily/817ab157404f4777810cf5cf5f196076 is 61, key is row-0/testFamily:testQualifier/1730982001882/Put/seqid=0 2024-11-07T12:21:32,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741920_1096 (size=5049) 2024-11-07T12:21:32,775 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:32,975 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-11-07T12:21:32,975 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1603): Online Regions={be85db67bdb4b8cbb0470630805ebc5d=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d., 6497436b1160bd1dd303db7791f75ae1=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1., 15982a8bd19de3fe41d3798072bfe2a3=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3., 0b0d536521b4033814c4e766b23cf3ae=hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.} 2024-11-07T12:21:32,975 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1, be85db67bdb4b8cbb0470630805ebc5d 2024-11-07T12:21:33,115 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/.tmp/testFamily/817ab157404f4777810cf5cf5f196076 2024-11-07T12:21:33,123 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/.tmp/testFamily/817ab157404f4777810cf5cf5f196076 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/testFamily/817ab157404f4777810cf5cf5f196076 2024-11-07T12:21:33,129 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/testFamily/817ab157404f4777810cf5cf5f196076, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:33,130 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be85db67bdb4b8cbb0470630805ebc5d in 436ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:33,133 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/be85db67bdb4b8cbb0470630805ebc5d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:33,134 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:21:33,134 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for be85db67bdb4b8cbb0470630805ebc5d: 2024-11-07T12:21:33,134 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730982000853.be85db67bdb4b8cbb0470630805ebc5d. 2024-11-07T12:21:33,134 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6497436b1160bd1dd303db7791f75ae1, disabling compactions & flushes 2024-11-07T12:21:33,134 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:21:33,134 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:21:33,134 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. after waiting 0 ms 2024-11-07T12:21:33,134 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:21:33,134 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6497436b1160bd1dd303db7791f75ae1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:33,150 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/.tmp/testFamily/ab34812550994a7496d2cfc79fdac532 is 61, key is row-0/testFamily:testQualifier/1730982054801/Put/seqid=0 2024-11-07T12:21:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741921_1097 (size=5049) 2024-11-07T12:21:33,176 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:21:33,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:21:33,376 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3, 6497436b1160bd1dd303db7791f75ae1 2024-11-07T12:21:33,555 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/.tmp/testFamily/ab34812550994a7496d2cfc79fdac532 2024-11-07T12:21:33,561 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/.tmp/testFamily/ab34812550994a7496d2cfc79fdac532 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/testFamily/ab34812550994a7496d2cfc79fdac532 2024-11-07T12:21:33,567 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/testFamily/ab34812550994a7496d2cfc79fdac532, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:33,568 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6497436b1160bd1dd303db7791f75ae1 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/6497436b1160bd1dd303db7791f75ae1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:33,572 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6497436b1160bd1dd303db7791f75ae1: 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1730982051784.6497436b1160bd1dd303db7791f75ae1. 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 15982a8bd19de3fe41d3798072bfe2a3, disabling compactions & flushes 2024-11-07T12:21:33,572 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. after waiting 0 ms 2024-11-07T12:21:33,572 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:21:33,572 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 15982a8bd19de3fe41d3798072bfe2a3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:33,576 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:21:33,588 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/.tmp/testFamily/7aa3bec6e59a4c45ab1cc9aa905e9c88 is 61, key is row-0/testFamily:testQualifier/1730981956906/Put/seqid=0 2024-11-07T12:21:33,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741922_1098 (size=5049) 2024-11-07T12:21:33,776 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:21:33,976 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-07T12:21:33,977 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1603): Online Regions={15982a8bd19de3fe41d3798072bfe2a3=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3., 0b0d536521b4033814c4e766b23cf3ae=hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae.} 2024-11-07T12:21:33,977 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae, 15982a8bd19de3fe41d3798072bfe2a3 2024-11-07T12:21:33,993 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/.tmp/testFamily/7aa3bec6e59a4c45ab1cc9aa905e9c88 2024-11-07T12:21:34,000 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/.tmp/testFamily/7aa3bec6e59a4c45ab1cc9aa905e9c88 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/testFamily/7aa3bec6e59a4c45ab1cc9aa905e9c88 2024-11-07T12:21:34,005 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/testFamily/7aa3bec6e59a4c45ab1cc9aa905e9c88, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:34,006 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 15982a8bd19de3fe41d3798072bfe2a3 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:34,010 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/15982a8bd19de3fe41d3798072bfe2a3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:34,011 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:21:34,011 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 15982a8bd19de3fe41d3798072bfe2a3: 2024-11-07T12:21:34,011 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730981953866.15982a8bd19de3fe41d3798072bfe2a3. 2024-11-07T12:21:34,011 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0b0d536521b4033814c4e766b23cf3ae, disabling compactions & flushes 2024-11-07T12:21:34,011 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:21:34,011 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:21:34,011 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. after waiting 0 ms 2024-11-07T12:21:34,011 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:21:34,011 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0b0d536521b4033814c4e766b23cf3ae 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T12:21:34,026 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/.tmp/testFamily/66bf60eabee9473cbbeb2367eca5fe67 is 61, key is row-0/testFamily:testQualifier/1730982024494/Put/seqid=0 2024-11-07T12:21:34,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741923_1099 (size=5049) 2024-11-07T12:21:34,177 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:21:34,377 DEBUG [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1629): Waiting on 0b0d536521b4033814c4e766b23cf3ae 2024-11-07T12:21:34,431 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/.tmp/testFamily/66bf60eabee9473cbbeb2367eca5fe67 2024-11-07T12:21:34,437 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/.tmp/testFamily/66bf60eabee9473cbbeb2367eca5fe67 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/testFamily/66bf60eabee9473cbbeb2367eca5fe67 2024-11-07T12:21:34,442 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/testFamily/66bf60eabee9473cbbeb2367eca5fe67, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T12:21:34,443 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0b0d536521b4033814c4e766b23cf3ae in 432ms, sequenceid=8, compaction requested=false 2024-11-07T12:21:34,447 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/data/hbase/testMetaScanTimeoutOnClose_1_-true/0b0d536521b4033814c4e766b23cf3ae/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T12:21:34,448 INFO [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:21:34,448 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0b0d536521b4033814c4e766b23cf3ae: 2024-11-07T12:21:34,448 DEBUG [RS_CLOSE_REGION-regionserver/fbad4c8a32e4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1730982023461.0b0d536521b4033814c4e766b23cf3ae. 2024-11-07T12:21:34,577 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1250): stopping server fbad4c8a32e4,44527,1730981867254; all regions closed. 2024-11-07T12:21:34,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741834_1010 (size=56731) 2024-11-07T12:21:34,588 DEBUG [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/oldWALs 2024-11-07T12:21:34,588 INFO [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL fbad4c8a32e4%2C44527%2C1730981867254.meta:.meta(num 1730981901782) 2024-11-07T12:21:34,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741833_1009 (size=60562) 2024-11-07T12:21:34,590 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/WALs/fbad4c8a32e4,44527,1730981867254/fbad4c8a32e4%2C44527%2C1730981867254.1730981901145 not finished, retry = 0 2024-11-07T12:21:34,693 DEBUG [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/oldWALs 2024-11-07T12:21:34,693 INFO [RS:0;fbad4c8a32e4:44527 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL fbad4c8a32e4%2C44527%2C1730981867254:(num 1730981901145) 2024-11-07T12:21:34,693 DEBUG [RS:0;fbad4c8a32e4:44527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:34,694 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:21:34,694 INFO [RS:0;fbad4c8a32e4:44527 {}] hbase.ChoreService(370): Chore service for: regionserver/fbad4c8a32e4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-11-07T12:21:34,694 INFO [regionserver/fbad4c8a32e4:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T12:21:34,695 INFO [RS:0;fbad4c8a32e4:44527 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44527 2024-11-07T12:21:34,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fbad4c8a32e4,44527,1730981867254 2024-11-07T12:21:34,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:21:34,701 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fbad4c8a32e4,44527,1730981867254] 2024-11-07T12:21:34,701 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing fbad4c8a32e4,44527,1730981867254; numProcessing=1 2024-11-07T12:21:34,703 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/fbad4c8a32e4,44527,1730981867254 already deleted, retry=false 2024-11-07T12:21:34,703 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; fbad4c8a32e4,44527,1730981867254 expired; onlineServers=0 2024-11-07T12:21:34,703 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'fbad4c8a32e4,36333,1730981865680' ***** 2024-11-07T12:21:34,703 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T12:21:34,703 DEBUG [M:0;fbad4c8a32e4:36333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264bae81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fbad4c8a32e4/172.17.0.2:0 2024-11-07T12:21:34,703 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegionServer(1224): stopping server fbad4c8a32e4,36333,1730981865680 2024-11-07T12:21:34,703 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegionServer(1250): stopping server fbad4c8a32e4,36333,1730981865680; all regions closed. 2024-11-07T12:21:34,703 DEBUG [M:0;fbad4c8a32e4:36333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:21:34,703 DEBUG [M:0;fbad4c8a32e4:36333 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T12:21:34,704 DEBUG [M:0;fbad4c8a32e4:36333 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T12:21:34,704 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T12:21:34,704 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster-HFileCleaner.small.0-1730981870645 {}] cleaner.HFileCleaner(306): Exit Thread[master/fbad4c8a32e4:0:becomeActiveMaster-HFileCleaner.small.0-1730981870645,5,FailOnTimeoutGroup] 2024-11-07T12:21:34,704 DEBUG [master/fbad4c8a32e4:0:becomeActiveMaster-HFileCleaner.large.0-1730981870615 {}] cleaner.HFileCleaner(306): Exit Thread[master/fbad4c8a32e4:0:becomeActiveMaster-HFileCleaner.large.0-1730981870615,5,FailOnTimeoutGroup] 2024-11-07T12:21:34,704 INFO [M:0;fbad4c8a32e4:36333 {}] hbase.ChoreService(370): Chore service for: master/fbad4c8a32e4:0 had [] on shutdown 2024-11-07T12:21:34,704 DEBUG [M:0;fbad4c8a32e4:36333 {}] master.HMaster(1733): Stopping service threads 2024-11-07T12:21:34,704 INFO [M:0;fbad4c8a32e4:36333 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T12:21:34,704 ERROR [M:0;fbad4c8a32e4:36333 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[LeaseRenewer:jenkins@localhost:33401,5,PEWorkerGroup] 2024-11-07T12:21:34,705 INFO [M:0;fbad4c8a32e4:36333 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T12:21:34,705 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T12:21:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T12:21:34,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:21:34,705 DEBUG [M:0;fbad4c8a32e4:36333 {}] zookeeper.ZKUtil(347): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T12:21:34,705 WARN [M:0;fbad4c8a32e4:36333 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T12:21:34,705 INFO [M:0;fbad4c8a32e4:36333 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-07T12:21:34,706 INFO [M:0;fbad4c8a32e4:36333 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T12:21:34,706 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T12:21:34,706 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:21:34,706 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:21:34,706 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:21:34,706 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:21:34,706 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:21:34,706 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.56 KB heapSize=520.16 KB 2024-11-07T12:21:34,724 DEBUG [M:0;fbad4c8a32e4:36333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/365f97aa311d40039e9ec674e32de9d3 is 82, key is hbase:meta,,1/info:regioninfo/1730981902037/Put/seqid=0 2024-11-07T12:21:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741924_1100 (size=5672) 2024-11-07T12:21:34,758 INFO [regionserver/fbad4c8a32e4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:21:34,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:21:34,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44527-0x10182aa98860001, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:21:34,801 INFO [RS:0;fbad4c8a32e4:44527 {}] regionserver.HRegionServer(1307): Exiting; stopping=fbad4c8a32e4,44527,1730981867254; zookeeper connection closed. 2024-11-07T12:21:34,801 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4234d17e {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4234d17e 2024-11-07T12:21:34,802 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T12:21:35,129 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/365f97aa311d40039e9ec674e32de9d3 2024-11-07T12:21:35,157 DEBUG [M:0;fbad4c8a32e4:36333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3584d4ecb3604b159fbcb41f9e5bc175 is 831, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1730981903914/Put/seqid=0 2024-11-07T12:21:35,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741925_1101 (size=30286) 2024-11-07T12:21:35,563 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=448.01 KB at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3584d4ecb3604b159fbcb41f9e5bc175 2024-11-07T12:21:35,568 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3584d4ecb3604b159fbcb41f9e5bc175 2024-11-07T12:21:35,583 DEBUG [M:0;fbad4c8a32e4:36333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/197a6b0f750b40b8b2c1f20e6bc1474f is 69, key is fbad4c8a32e4,44527,1730981867254/rs:state/1730981870681/Put/seqid=0 2024-11-07T12:21:35,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741926_1102 (size=5156) 2024-11-07T12:21:35,988 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/197a6b0f750b40b8b2c1f20e6bc1474f 2024-11-07T12:21:35,994 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/365f97aa311d40039e9ec674e32de9d3 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/365f97aa311d40039e9ec674e32de9d3 2024-11-07T12:21:36,000 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/365f97aa311d40039e9ec674e32de9d3, entries=8, sequenceid=832, filesize=5.5 K 2024-11-07T12:21:36,000 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3584d4ecb3604b159fbcb41f9e5bc175 as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3584d4ecb3604b159fbcb41f9e5bc175 2024-11-07T12:21:36,005 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3584d4ecb3604b159fbcb41f9e5bc175 2024-11-07T12:21:36,005 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3584d4ecb3604b159fbcb41f9e5bc175, entries=92, sequenceid=832, filesize=29.6 K 2024-11-07T12:21:36,007 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/197a6b0f750b40b8b2c1f20e6bc1474f as hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/197a6b0f750b40b8b2c1f20e6bc1474f 2024-11-07T12:21:36,011 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33401/user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/197a6b0f750b40b8b2c1f20e6bc1474f, entries=1, sequenceid=832, filesize=5.0 K 2024-11-07T12:21:36,013 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.56 KB/459328, heapSize ~519.87 KB/532344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1306ms, sequenceid=832, compaction requested=false 2024-11-07T12:21:36,014 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:21:36,014 DEBUG [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T12:21:36,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39577 is added to blk_1073741830_1006 (size=514327) 2024-11-07T12:21:36,017 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/f86027cc-2413-b30a-ef07-74e02a3317bc/MasterData/WALs/fbad4c8a32e4,36333,1730981865680/fbad4c8a32e4%2C36333%2C1730981865680.1730981869195 not finished, retry = 0 2024-11-07T12:21:36,118 INFO [M:0;fbad4c8a32e4:36333 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-07T12:21:36,118 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T12:21:36,118 INFO [M:0;fbad4c8a32e4:36333 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36333 2024-11-07T12:21:36,120 DEBUG [M:0;fbad4c8a32e4:36333 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/fbad4c8a32e4,36333,1730981865680 already deleted, retry=false 2024-11-07T12:21:36,222 INFO [M:0;fbad4c8a32e4:36333 {}] regionserver.HRegionServer(1307): Exiting; stopping=fbad4c8a32e4,36333,1730981865680; zookeeper connection closed. 2024-11-07T12:21:36,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:21:36,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36333-0x10182aa98860000, quorum=127.0.0.1:51230, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:21:36,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b30907c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:21:36,228 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2aa1746d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:21:36,228 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:21:36,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cf34ac6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:21:36,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@318641df{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/hadoop.log.dir/,STOPPED} 2024-11-07T12:21:36,231 WARN [BP-1253633137-172.17.0.2-1730981859271 heartbeating to localhost/127.0.0.1:33401 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T12:21:36,231 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T12:21:36,231 WARN [BP-1253633137-172.17.0.2-1730981859271 heartbeating to localhost/127.0.0.1:33401 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1253633137-172.17.0.2-1730981859271 (Datanode Uuid 83c6d118-22a1-4447-8a00-68f28783d20d) service to localhost/127.0.0.1:33401 2024-11-07T12:21:36,231 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T12:21:36,232 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/dfs/data/data1/current/BP-1253633137-172.17.0.2-1730981859271 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:21:36,233 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/cluster_a71d3a03-90c2-e61f-790b-7fcccfb7a1e1/dfs/data/data2/current/BP-1253633137-172.17.0.2-1730981859271 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:21:36,233 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T12:21:36,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4f422bdf{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:21:36,240 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c457aa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:21:36,240 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:21:36,240 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4803ff98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:21:36,240 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1857c056{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/2f115220-75ba-f13a-1265-3cfb7ea556db/hadoop.log.dir/,STOPPED} 2024-11-07T12:21:36,251 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-07T12:21:36,286 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down