2024-11-08 10:18:09,090 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 10:18:09,106 main DEBUG Took 0.013823 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 10:18:09,106 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 10:18:09,107 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 10:18:09,108 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 10:18:09,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,153 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 10:18:09,171 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,174 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,176 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,176 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,178 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,179 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,181 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,182 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,183 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,185 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,187 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,188 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,189 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:18:09,190 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,191 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 10:18:09,193 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:18:09,195 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 10:18:09,206 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 10:18:09,207 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 10:18:09,209 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 10:18:09,209 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 10:18:09,222 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 10:18:09,226 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 10:18:09,229 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 10:18:09,229 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 10:18:09,230 main DEBUG createAppenders(={Console}) 2024-11-08 10:18:09,231 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-08 10:18:09,231 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 10:18:09,231 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-08 10:18:09,232 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 10:18:09,233 main DEBUG OutputStream closed 2024-11-08 10:18:09,233 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 10:18:09,233 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 10:18:09,234 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-08 10:18:09,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 10:18:09,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 10:18:09,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 10:18:09,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 10:18:09,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 10:18:09,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 10:18:09,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 10:18:09,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 10:18:09,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 10:18:09,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 10:18:09,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 10:18:09,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 10:18:09,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 10:18:09,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 10:18:09,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 10:18:09,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 10:18:09,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 10:18:09,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 10:18:09,440 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 10:18:09,442 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-08 10:18:09,443 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 10:18:09,444 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-08T10:18:09,747 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-08 10:18:09,754 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 10:18:09,758 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T10:18:09,860 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7 2024-11-08T10:18:09,914 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T10:18:09,917 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b, deleteOnExit=true 2024-11-08T10:18:09,918 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-08T10:18:09,919 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/test.cache.data in system properties and HBase conf 2024-11-08T10:18:09,919 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T10:18:09,920 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/hadoop.log.dir in system properties and HBase conf 2024-11-08T10:18:09,921 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T10:18:09,922 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T10:18:09,922 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-08T10:18:10,056 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T10:18:10,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:18:10,191 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T10:18:10,191 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T10:18:10,192 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T10:18:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T10:18:10,193 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T10:18:10,194 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T10:18:10,195 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T10:18:10,195 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T10:18:10,196 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T10:18:10,197 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/nfs.dump.dir in system properties and HBase conf 2024-11-08T10:18:10,197 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/java.io.tmpdir in system properties and HBase conf 2024-11-08T10:18:10,198 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T10:18:10,199 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T10:18:10,199 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T10:18:11,396 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T10:18:11,551 INFO [Time-limited test {}] log.Log(170): Logging initialized @3560ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T10:18:11,664 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T10:18:11,755 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T10:18:11,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T10:18:11,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T10:18:11,791 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T10:18:11,808 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T10:18:11,811 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c68af8f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/hadoop.log.dir/,AVAILABLE} 2024-11-08T10:18:11,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d3a9ea5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T10:18:12,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1af42a04{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/java.io.tmpdir/jetty-localhost-37683-hadoop-hdfs-3_4_1-tests_jar-_-any-16026658957078594168/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T10:18:12,126 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8a3c7b8{HTTP/1.1, (http/1.1)}{localhost:37683} 2024-11-08T10:18:12,126 INFO [Time-limited test {}] server.Server(415): Started @4137ms 2024-11-08T10:18:12,710 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T10:18:12,720 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T10:18:12,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T10:18:12,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T10:18:12,737 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T10:18:12,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ed11b40{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/hadoop.log.dir/,AVAILABLE} 2024-11-08T10:18:12,739 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c8914a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T10:18:12,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ae43da4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/java.io.tmpdir/jetty-localhost-36387-hadoop-hdfs-3_4_1-tests_jar-_-any-15044845564972801771/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T10:18:12,892 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@540cee33{HTTP/1.1, (http/1.1)}{localhost:36387} 2024-11-08T10:18:12,892 INFO [Time-limited test {}] server.Server(415): Started @4903ms 2024-11-08T10:18:12,967 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T10:18:13,681 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/dfs/data/data1/current/BP-1631354101-172.17.0.2-1731061091046/current, will proceed with Du for space computation calculation, 2024-11-08T10:18:13,681 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/dfs/data/data2/current/BP-1631354101-172.17.0.2-1731061091046/current, will proceed with Du for space computation calculation, 2024-11-08T10:18:13,751 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T10:18:13,819 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9bfefa8adb2a7538 with lease ID 0xffd754e0c57a9216: Processing first storage report for DS-342be98b-0954-463b-93c2-30fbec34c373 from datanode DatanodeRegistration(127.0.0.1:37041, datanodeUuid=9f7de0e8-95e7-42c9-b2b6-c2442a647b59, infoPort=41557, infoSecurePort=0, ipcPort=43033, storageInfo=lv=-57;cid=testClusterID;nsid=1045733654;c=1731061091046) 2024-11-08T10:18:13,820 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9bfefa8adb2a7538 with lease ID 0xffd754e0c57a9216: from storage DS-342be98b-0954-463b-93c2-30fbec34c373 node DatanodeRegistration(127.0.0.1:37041, datanodeUuid=9f7de0e8-95e7-42c9-b2b6-c2442a647b59, infoPort=41557, infoSecurePort=0, ipcPort=43033, storageInfo=lv=-57;cid=testClusterID;nsid=1045733654;c=1731061091046), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T10:18:13,821 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9bfefa8adb2a7538 with lease ID 0xffd754e0c57a9216: Processing first storage report for DS-8bfe136e-8be3-4cc3-8001-5357310f163d from datanode DatanodeRegistration(127.0.0.1:37041, datanodeUuid=9f7de0e8-95e7-42c9-b2b6-c2442a647b59, infoPort=41557, infoSecurePort=0, ipcPort=43033, storageInfo=lv=-57;cid=testClusterID;nsid=1045733654;c=1731061091046) 2024-11-08T10:18:13,821 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9bfefa8adb2a7538 with lease ID 0xffd754e0c57a9216: from storage DS-8bfe136e-8be3-4cc3-8001-5357310f163d node DatanodeRegistration(127.0.0.1:37041, datanodeUuid=9f7de0e8-95e7-42c9-b2b6-c2442a647b59, infoPort=41557, infoSecurePort=0, ipcPort=43033, storageInfo=lv=-57;cid=testClusterID;nsid=1045733654;c=1731061091046), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T10:18:13,893 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7 2024-11-08T10:18:14,001 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/zookeeper_0, clientPort=50547, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T10:18:14,014 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50547 2024-11-08T10:18:14,029 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:14,037 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:14,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741825_1001 (size=7) 2024-11-08T10:18:14,392 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 with version=8 2024-11-08T10:18:14,392 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/hbase-staging 2024-11-08T10:18:14,582 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T10:18:14,932 INFO [Time-limited test {}] client.ConnectionUtils(129): master/2db67accb017:0 server-side Connection retries=9 2024-11-08T10:18:14,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:18:14,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:18:14,953 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:18:14,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:18:14,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:18:15,149 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:18:15,232 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T10:18:15,243 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T10:18:15,247 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:18:15,282 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14036 (auto-detected) 2024-11-08T10:18:15,284 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T10:18:15,312 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40511 2024-11-08T10:18:15,335 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:15,338 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:15,359 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40511 connecting to ZooKeeper ensemble=127.0.0.1:50547 2024-11-08T10:18:15,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405110x0, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:18:15,413 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40511-0x10187638bc40000 connected 2024-11-08T10:18:15,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:18:15,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:18:15,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:18:15,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40511 2024-11-08T10:18:15,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40511 2024-11-08T10:18:15,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40511 2024-11-08T10:18:15,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40511 2024-11-08T10:18:15,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40511 2024-11-08T10:18:15,498 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01, hbase.cluster.distributed=false 2024-11-08T10:18:15,591 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/2db67accb017:0 server-side Connection retries=9 2024-11-08T10:18:15,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:18:15,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:18:15,592 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:18:15,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:18:15,593 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:18:15,595 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:18:15,599 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:18:15,600 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:37913 2024-11-08T10:18:15,603 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T10:18:15,623 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T10:18:15,627 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:15,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:15,643 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37913 connecting to ZooKeeper ensemble=127.0.0.1:50547 2024-11-08T10:18:15,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379130x0, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:18:15,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379130x0, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:18:15,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379130x0, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:18:15,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379130x0, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:18:15,674 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37913-0x10187638bc40001 connected 2024-11-08T10:18:15,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37913 2024-11-08T10:18:15,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37913 2024-11-08T10:18:15,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37913 2024-11-08T10:18:15,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37913 2024-11-08T10:18:15,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37913 2024-11-08T10:18:15,701 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/2db67accb017,40511,1731061094572 2024-11-08T10:18:15,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:18:15,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:18:15,721 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2db67accb017,40511,1731061094572 2024-11-08T10:18:15,726 DEBUG [M:0;2db67accb017:40511 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2db67accb017:40511 2024-11-08T10:18:15,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T10:18:15,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T10:18:15,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:15,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:15,752 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T10:18:15,753 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T10:18:15,754 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2db67accb017,40511,1731061094572 from backup master directory 2024-11-08T10:18:15,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:18:15,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2db67accb017,40511,1731061094572 2024-11-08T10:18:15,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:18:15,760 WARN [master/2db67accb017:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:18:15,760 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2db67accb017,40511,1731061094572 2024-11-08T10:18:15,763 INFO [master/2db67accb017:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T10:18:15,776 INFO [master/2db67accb017:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T10:18:15,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741826_1002 (size=42) 2024-11-08T10:18:15,875 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/hbase.id with ID: 5b7762d5-67ab-46ed-b4d2-da0d9bd30faf 2024-11-08T10:18:15,937 INFO [master/2db67accb017:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T10:18:15,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:15,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:15,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741827_1003 (size=196) 2024-11-08T10:18:16,011 INFO [master/2db67accb017:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:18:16,014 INFO [master/2db67accb017:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T10:18:16,037 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:18:16,047 INFO [master/2db67accb017:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:18:16,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741828_1004 (size=1189) 2024-11-08T10:18:16,155 INFO [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store 2024-11-08T10:18:16,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741829_1005 (size=34) 2024-11-08T10:18:16,191 INFO [master/2db67accb017:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T10:18:16,191 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:16,193 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T10:18:16,193 INFO [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:18:16,193 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:18:16,194 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T10:18:16,194 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:18:16,194 INFO [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:18:16,194 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T10:18:16,198 WARN [master/2db67accb017:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/.initializing 2024-11-08T10:18:16,199 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/WALs/2db67accb017,40511,1731061094572 2024-11-08T10:18:16,210 INFO [master/2db67accb017:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:18:16,224 INFO [master/2db67accb017:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2db67accb017%2C40511%2C1731061094572, suffix=, logDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/WALs/2db67accb017,40511,1731061094572, archiveDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/oldWALs, maxLogs=10 2024-11-08T10:18:16,253 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/WALs/2db67accb017,40511,1731061094572/2db67accb017%2C40511%2C1731061094572.1731061096230, exclude list is [], retry=0 2024-11-08T10:18:16,272 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-342be98b-0954-463b-93c2-30fbec34c373,DISK] 2024-11-08T10:18:16,275 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T10:18:16,319 INFO [master/2db67accb017:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/WALs/2db67accb017,40511,1731061094572/2db67accb017%2C40511%2C1731061094572.1731061096230 2024-11-08T10:18:16,320 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41557:41557)] 2024-11-08T10:18:16,321 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:18:16,321 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:16,325 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,326 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T10:18:16,416 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:16,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:16,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T10:18:16,426 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:16,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:18:16,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T10:18:16,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:16,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:18:16,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T10:18:16,437 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:16,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:18:16,443 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,444 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,455 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:18:16,459 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:18:16,466 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:18:16,467 INFO [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64835000, jitterRate=-0.03388321399688721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:18:16,472 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T10:18:16,473 INFO [master/2db67accb017:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T10:18:16,508 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c371d1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:16,553 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-08T10:18:16,569 INFO [master/2db67accb017:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T10:18:16,569 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T10:18:16,572 INFO [master/2db67accb017:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T10:18:16,574 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T10:18:16,579 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 5 msec 2024-11-08T10:18:16,579 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T10:18:16,610 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T10:18:16,630 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T10:18:16,632 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-08T10:18:16,635 INFO [master/2db67accb017:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T10:18:16,637 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T10:18:16,638 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-08T10:18:16,640 INFO [master/2db67accb017:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T10:18:16,645 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T10:18:16,646 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-08T10:18:16,648 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T10:18:16,650 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T10:18:16,663 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T10:18:16,665 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T10:18:16,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:18:16,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:16,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:18:16,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:16,672 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=2db67accb017,40511,1731061094572, sessionid=0x10187638bc40000, setting cluster-up flag (Was=false) 2024-11-08T10:18:16,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:16,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:16,697 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T10:18:16,700 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2db67accb017,40511,1731061094572 2024-11-08T10:18:16,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:16,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:16,715 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T10:18:16,717 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2db67accb017,40511,1731061094572 2024-11-08T10:18:16,820 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2db67accb017:37913 2024-11-08T10:18:16,822 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1008): ClusterId : 5b7762d5-67ab-46ed-b4d2-da0d9bd30faf 2024-11-08T10:18:16,827 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T10:18:16,832 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-08T10:18:16,836 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T10:18:16,836 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T10:18:16,839 INFO [master/2db67accb017:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-08T10:18:16,840 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T10:18:16,840 DEBUG [RS:0;2db67accb017:37913 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341f6143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:16,843 INFO [master/2db67accb017:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T10:18:16,848 DEBUG [RS:0;2db67accb017:37913 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8a7fb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2db67accb017/172.17.0.2:0 2024-11-08T10:18:16,849 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2db67accb017,40511,1731061094572 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T10:18:16,852 INFO [RS:0;2db67accb017:37913 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-08T10:18:16,852 INFO [RS:0;2db67accb017:37913 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-08T10:18:16,852 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-08T10:18:16,853 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2db67accb017:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:18:16,853 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2db67accb017:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:18:16,853 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2db67accb017:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:18:16,853 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2db67accb017:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:18:16,854 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2db67accb017:0, corePoolSize=10, maxPoolSize=10 2024-11-08T10:18:16,854 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:16,854 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2db67accb017:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:18:16,855 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:16,855 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3073): reportForDuty to master=2db67accb017,40511,1731061094572 with isa=2db67accb017/172.17.0.2:37913, startcode=1731061095589 2024-11-08T10:18:16,859 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731061126859 2024-11-08T10:18:16,862 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T10:18:16,861 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:18:16,862 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-08T10:18:16,863 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T10:18:16,867 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T10:18:16,867 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:16,868 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T10:18:16,868 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T10:18:16,868 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T10:18:16,869 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:16,870 DEBUG [RS:0;2db67accb017:37913 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:18:16,868 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T10:18:16,873 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T10:18:16,875 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T10:18:16,875 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T10:18:16,881 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T10:18:16,881 INFO [master/2db67accb017:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T10:18:16,884 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2db67accb017:0:becomeActiveMaster-HFileCleaner.large.0-1731061096882,5,FailOnTimeoutGroup] 2024-11-08T10:18:16,889 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2db67accb017:0:becomeActiveMaster-HFileCleaner.small.0-1731061096884,5,FailOnTimeoutGroup] 2024-11-08T10:18:16,890 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:16,890 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T10:18:16,891 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:16,892 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:16,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741831_1007 (size=1039) 2024-11-08T10:18:16,901 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-08T10:18:16,902 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:18:16,923 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52997, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:18:16,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40511 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 2db67accb017,37913,1731061095589 2024-11-08T10:18:16,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40511 {}] master.ServerManager(486): Registering regionserver=2db67accb017,37913,1731061095589 2024-11-08T10:18:16,955 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:18:16,955 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:46031 2024-11-08T10:18:16,955 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-08T10:18:16,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:18:16,963 DEBUG [RS:0;2db67accb017:37913 {}] zookeeper.ZKUtil(111): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2db67accb017,37913,1731061095589 2024-11-08T10:18:16,963 WARN [RS:0;2db67accb017:37913 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:18:16,963 INFO [RS:0;2db67accb017:37913 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:18:16,964 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589 2024-11-08T10:18:16,973 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2db67accb017,37913,1731061095589] 2024-11-08T10:18:16,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741832_1008 (size=32) 2024-11-08T10:18:17,001 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-08T10:18:17,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:17,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T10:18:17,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:18:17,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:17,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:17,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T10:18:17,018 INFO [RS:0;2db67accb017:37913 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T10:18:17,019 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:18:17,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:17,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:17,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T10:18:17,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:18:17,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:17,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:17,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740 2024-11-08T10:18:17,044 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740 2024-11-08T10:18:17,046 INFO [RS:0;2db67accb017:37913 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T10:18:17,053 INFO [RS:0;2db67accb017:37913 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T10:18:17,054 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,058 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T10:18:17,061 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-08T10:18:17,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T10:18:17,070 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,070 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,071 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,071 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,074 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,074 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,074 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2db67accb017:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:18:17,074 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,075 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,075 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,075 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,075 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2db67accb017:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:18:17,076 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2db67accb017:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:18:17,076 DEBUG [RS:0;2db67accb017:37913 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2db67accb017:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:18:17,082 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:18:17,083 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73014562, jitterRate=0.08800175786018372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T10:18:17,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T10:18:17,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:18:17,087 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T10:18:17,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T10:18:17,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:18:17,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:18:17,104 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,105 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,105 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,105 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T10:18:17,105 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,105 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T10:18:17,105 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,37913,1731061095589-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:18:17,110 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:18:17,110 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-08T10:18:17,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T10:18:17,129 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:18:17,133 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T10:18:17,145 INFO [RS:0;2db67accb017:37913 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T10:18:17,149 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,37913,1731061095589-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:17,183 INFO [RS:0;2db67accb017:37913 {}] regionserver.Replication(204): 2db67accb017,37913,1731061095589 started 2024-11-08T10:18:17,183 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1767): Serving as 2db67accb017,37913,1731061095589, RpcServer on 2db67accb017/172.17.0.2:37913, sessionid=0x10187638bc40001 2024-11-08T10:18:17,184 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T10:18:17,184 DEBUG [RS:0;2db67accb017:37913 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2db67accb017,37913,1731061095589 2024-11-08T10:18:17,184 DEBUG [RS:0;2db67accb017:37913 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2db67accb017,37913,1731061095589' 2024-11-08T10:18:17,184 DEBUG [RS:0;2db67accb017:37913 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T10:18:17,185 DEBUG [RS:0;2db67accb017:37913 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T10:18:17,186 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T10:18:17,186 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T10:18:17,186 DEBUG [RS:0;2db67accb017:37913 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2db67accb017,37913,1731061095589 2024-11-08T10:18:17,187 DEBUG [RS:0;2db67accb017:37913 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2db67accb017,37913,1731061095589' 2024-11-08T10:18:17,187 DEBUG [RS:0;2db67accb017:37913 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T10:18:17,187 DEBUG [RS:0;2db67accb017:37913 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T10:18:17,188 DEBUG [RS:0;2db67accb017:37913 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T10:18:17,188 INFO [RS:0;2db67accb017:37913 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T10:18:17,188 INFO [RS:0;2db67accb017:37913 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T10:18:17,284 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:18,285 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:19,286 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:20,286 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:21,287 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:22,288 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:23,289 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:24,289 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:25,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T10:18:25,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T10:18:25,290 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:26,291 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:27,291 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:28,292 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:29,293 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:30,294 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:31,294 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:32,295 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:33,296 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:34,296 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:35,297 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:36,297 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:37,298 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:38,299 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:39,300 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:40,300 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:41,301 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:42,302 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:43,302 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:43,887 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:18:44,153 ERROR [LeaseRenewer:jenkins@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46031,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:18:44,303 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:45,304 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:46,304 WARN [2db67accb017:40511 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:18:47,197 INFO [RS:0;2db67accb017:37913 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:18:47,209 INFO [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2db67accb017%2C37913%2C1731061095589, suffix=, logDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589, archiveDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/oldWALs, maxLogs=32 2024-11-08T10:18:47,236 DEBUG [RS:0;2db67accb017:37913 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589/2db67accb017%2C37913%2C1731061095589.1731061127214, exclude list is [], retry=0 2024-11-08T10:18:47,243 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-342be98b-0954-463b-93c2-30fbec34c373,DISK] 2024-11-08T10:18:47,247 INFO [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589/2db67accb017%2C37913%2C1731061095589.1731061127214 2024-11-08T10:18:47,248 DEBUG [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41557:41557)] 2024-11-08T10:18:47,306 DEBUG [2db67accb017:40511 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T10:18:47,312 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:47,318 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2db67accb017,37913,1731061095589, state=OPENING 2024-11-08T10:18:47,324 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T10:18:47,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:47,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:47,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:18:47,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:18:47,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:18:47,510 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:18:47,512 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T10:18:47,517 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54764, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T10:18:47,531 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-08T10:18:47,531 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:18:47,532 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T10:18:47,539 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2db67accb017%2C37913%2C1731061095589.meta, suffix=.meta, logDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589, archiveDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/oldWALs, maxLogs=32 2024-11-08T10:18:47,560 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589/2db67accb017%2C37913%2C1731061095589.meta.1731061127542.meta, exclude list is [], retry=0 2024-11-08T10:18:47,565 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37041,DS-342be98b-0954-463b-93c2-30fbec34c373,DISK] 2024-11-08T10:18:47,570 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/WALs/2db67accb017,37913,1731061095589/2db67accb017%2C37913%2C1731061095589.meta.1731061127542.meta 2024-11-08T10:18:47,580 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41557:41557)] 2024-11-08T10:18:47,580 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:18:47,582 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T10:18:47,655 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T10:18:47,661 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T10:18:47,667 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T10:18:47,667 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:47,667 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-08T10:18:47,668 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-08T10:18:47,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T10:18:47,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:18:47,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:47,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:47,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T10:18:47,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:18:47,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:47,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:47,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T10:18:47,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:18:47,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:47,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:18:47,687 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740 2024-11-08T10:18:47,690 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740 2024-11-08T10:18:47,695 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T10:18:47,698 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T10:18:47,701 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62320487, jitterRate=-0.07135237753391266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T10:18:47,703 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T10:18:47,712 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731061127503 2024-11-08T10:18:47,727 DEBUG [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T10:18:47,728 INFO [RS_OPEN_META-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-08T10:18:47,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:47,732 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2db67accb017,37913,1731061095589, state=OPEN 2024-11-08T10:18:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:18:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:18:47,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:18:47,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:18:47,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T10:18:47,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=2db67accb017,37913,1731061095589 in 407 msec 2024-11-08T10:18:47,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T10:18:47,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.6290 sec 2024-11-08T10:18:47,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 30.9860 sec 2024-11-08T10:18:47,762 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731061127762, completionTime=-1 2024-11-08T10:18:47,762 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T10:18:47,762 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-08T10:18:47,807 DEBUG [hconnection-0x63d7c5b5-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:18:47,810 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54780, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:18:47,823 INFO [master/2db67accb017:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-08T10:18:47,823 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731061187823 2024-11-08T10:18:47,823 INFO [master/2db67accb017:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731061247823 2024-11-08T10:18:47,823 INFO [master/2db67accb017:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 60 msec 2024-11-08T10:18:47,844 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:18:47,847 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59230, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:18:47,852 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,40511,1731061094572-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:47,852 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,40511,1731061094572-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:47,853 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,40511,1731061094572-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:47,855 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2db67accb017:40511, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:47,855 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:47,862 DEBUG [master/2db67accb017:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-08T10:18:47,865 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-08T10:18:47,866 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T10:18:47,875 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-08T10:18:47,879 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:18:47,880 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:47,883 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:18:47,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741835_1011 (size=358) 2024-11-08T10:18:48,308 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c98f706b3d1970465cfb9fe302b92eeb, NAME => 'hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:18:48,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741836_1012 (size=42) 2024-11-08T10:18:48,326 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:48,326 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing c98f706b3d1970465cfb9fe302b92eeb, disabling compactions & flushes 2024-11-08T10:18:48,326 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,326 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,326 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. after waiting 0 ms 2024-11-08T10:18:48,326 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,327 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,327 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for c98f706b3d1970465cfb9fe302b92eeb: 2024-11-08T10:18:48,330 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:18:48,338 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731061128331"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061128331"}]},"ts":"1731061128331"} 2024-11-08T10:18:48,369 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:18:48,373 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:18:48,377 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061128373"}]},"ts":"1731061128373"} 2024-11-08T10:18:48,387 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-08T10:18:48,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=c98f706b3d1970465cfb9fe302b92eeb, ASSIGN}] 2024-11-08T10:18:48,398 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=c98f706b3d1970465cfb9fe302b92eeb, ASSIGN 2024-11-08T10:18:48,401 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=c98f706b3d1970465cfb9fe302b92eeb, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:18:48,552 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=c98f706b3d1970465cfb9fe302b92eeb, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:48,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure c98f706b3d1970465cfb9fe302b92eeb, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:18:48,714 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:18:48,724 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,725 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => c98f706b3d1970465cfb9fe302b92eeb, NAME => 'hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:18:48,725 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,726 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:48,726 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,726 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,729 INFO [StoreOpener-c98f706b3d1970465cfb9fe302b92eeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,732 INFO [StoreOpener-c98f706b3d1970465cfb9fe302b92eeb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98f706b3d1970465cfb9fe302b92eeb columnFamilyName info 2024-11-08T10:18:48,732 DEBUG [StoreOpener-c98f706b3d1970465cfb9fe302b92eeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:48,736 INFO [StoreOpener-c98f706b3d1970465cfb9fe302b92eeb-1 {}] regionserver.HStore(327): Store=c98f706b3d1970465cfb9fe302b92eeb/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:18:48,738 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,739 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:18:48,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:18:48,758 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened c98f706b3d1970465cfb9fe302b92eeb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70570672, jitterRate=0.05158495903015137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:18:48,760 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for c98f706b3d1970465cfb9fe302b92eeb: 2024-11-08T10:18:48,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb., pid=6, masterSystemTime=1731061128714 2024-11-08T10:18:48,769 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,769 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:18:48,771 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=c98f706b3d1970465cfb9fe302b92eeb, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:48,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T10:18:48,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure c98f706b3d1970465cfb9fe302b92eeb, server=2db67accb017,37913,1731061095589 in 218 msec 2024-11-08T10:18:48,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T10:18:48,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=c98f706b3d1970465cfb9fe302b92eeb, ASSIGN in 387 msec 2024-11-08T10:18:48,787 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:18:48,788 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061128787"}]},"ts":"1731061128787"} 2024-11-08T10:18:48,792 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-08T10:18:48,797 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:18:48,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 931 msec 2024-11-08T10:18:48,893 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-08T10:18:48,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-08T10:18:48,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:18:48,954 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-08T10:18:48,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T10:18:48,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 44 msec 2024-11-08T10:18:49,002 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-08T10:18:49,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T10:18:49,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 29 msec 2024-11-08T10:18:49,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-08T10:18:49,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-08T10:18:49,052 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 33.292sec 2024-11-08T10:18:49,053 INFO [master/2db67accb017:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T10:18:49,055 INFO [master/2db67accb017:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T10:18:49,056 INFO [master/2db67accb017:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T10:18:49,057 INFO [master/2db67accb017:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T10:18:49,057 INFO [master/2db67accb017:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T10:18:49,058 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,40511,1731061094572-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:18:49,059 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,40511,1731061094572-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T10:18:49,070 DEBUG [master/2db67accb017:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-08T10:18:49,071 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T10:18:49,071 INFO [master/2db67accb017:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2db67accb017,40511,1731061094572-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:18:49,153 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60b11d5f to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@455cf704 2024-11-08T10:18:49,160 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-08T10:18:49,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2615fbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:49,181 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T10:18:49,182 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T10:18:49,204 DEBUG [hconnection-0x2e97f5f2-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:18:49,225 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54782, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:18:49,239 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=2db67accb017,40511,1731061094572 2024-11-08T10:18:49,276 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=219, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=2937 2024-11-08T10:18:49,282 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x050886eb to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@756b37fe 2024-11-08T10:18:49,295 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x050886eb {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3cb45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:49,304 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x39cf731a to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1537671a 2024-11-08T10:18:49,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc9718f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:49,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T10:18:49,363 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59238, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T10:18:49,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:18:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T10:18:49,392 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:18:49,393 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:49,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:18:49,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-08T10:18:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T10:18:49,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741837_1013 (size=437) 2024-11-08T10:18:49,497 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => dc59aab9dd788edfed6060a8dd176fad, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:18:49,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741838_1014 (size=88) 2024-11-08T10:18:49,537 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:49,537 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing dc59aab9dd788edfed6060a8dd176fad, disabling compactions & flushes 2024-11-08T10:18:49,537 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,537 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,537 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. after waiting 0 ms 2024-11-08T10:18:49,537 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,537 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,537 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for dc59aab9dd788edfed6060a8dd176fad: 2024-11-08T10:18:49,540 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:18:49,540 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731061129540"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061129540"}]},"ts":"1731061129540"} 2024-11-08T10:18:49,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:18:49,548 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:18:49,549 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061129548"}]},"ts":"1731061129548"} 2024-11-08T10:18:49,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:18:49,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dc59aab9dd788edfed6060a8dd176fad, ASSIGN}] 2024-11-08T10:18:49,563 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dc59aab9dd788edfed6060a8dd176fad, ASSIGN 2024-11-08T10:18:49,564 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dc59aab9dd788edfed6060a8dd176fad, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:18:49,715 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=dc59aab9dd788edfed6060a8dd176fad, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:49,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure dc59aab9dd788edfed6060a8dd176fad, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:18:49,880 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:18:49,888 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,889 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => dc59aab9dd788edfed6060a8dd176fad, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:18:49,889 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,889 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:49,890 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,890 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,893 INFO [StoreOpener-dc59aab9dd788edfed6060a8dd176fad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,896 INFO [StoreOpener-dc59aab9dd788edfed6060a8dd176fad-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc59aab9dd788edfed6060a8dd176fad columnFamilyName testFamily 2024-11-08T10:18:49,896 DEBUG [StoreOpener-dc59aab9dd788edfed6060a8dd176fad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:49,898 INFO [StoreOpener-dc59aab9dd788edfed6060a8dd176fad-1 {}] regionserver.HStore(327): Store=dc59aab9dd788edfed6060a8dd176fad/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:18:49,900 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,901 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,905 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:18:49,911 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:18:49,912 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened dc59aab9dd788edfed6060a8dd176fad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70742060, jitterRate=0.05413883924484253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:18:49,914 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for dc59aab9dd788edfed6060a8dd176fad: 2024-11-08T10:18:49,916 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad., pid=11, masterSystemTime=1731061129880 2024-11-08T10:18:49,920 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,920 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:18:49,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=dc59aab9dd788edfed6060a8dd176fad, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:49,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T10:18:49,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure dc59aab9dd788edfed6060a8dd176fad, server=2db67accb017,37913,1731061095589 in 205 msec 2024-11-08T10:18:49,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T10:18:49,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dc59aab9dd788edfed6060a8dd176fad, ASSIGN in 371 msec 2024-11-08T10:18:49,939 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:18:49,939 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061129939"}]},"ts":"1731061129939"} 2024-11-08T10:18:49,942 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:18:49,956 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:18:49,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 575 msec 2024-11-08T10:18:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T10:18:50,483 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-08T10:18:50,483 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:18:50,485 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:18:50,493 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T10:18:50,494 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:18:50,494 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-08T10:18:50,497 DEBUG [hconnection-0x49e5c8d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:18:50,500 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42982, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:18:50,516 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:18:50,522 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T10:18:50,539 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:18:50,543 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x050886eb {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:18:50,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:18:50,549 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42984, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:18:50,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:18:50,561 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T10:18:50,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463627 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:18:50,565 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T10:18:50,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463627 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:18:50,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087989 2024-11-08T10:18:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42984 deadline: 1731061133564, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:18:50,595 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463627 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:18:50,600 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T10:18:50,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463628 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:18:50,602 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T10:18:50,602 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:18:50,604 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:18:50,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463629 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:18:50,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087987 2024-11-08T10:18:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42984 deadline: 1731061133603, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:18:50,607 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463629 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:18:50,609 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:18:50,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463630 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:18:50,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-4907804905177087986 2024-11-08T10:18:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42984 deadline: 1731061133608, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:18:50,612 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463630 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:18:50,616 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:18:50,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463631 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:18:50,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-4907804905177087985 2024-11-08T10:18:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42984 deadline: 1731061133615, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:18:50,622 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463631 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:18:50,624 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:18:50,625 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x39cf731a to 127.0.0.1:50547 2024-11-08T10:18:50,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:18:50,626 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:18:50,626 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:18:50,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:18:50,633 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x050886eb to 127.0.0.1:50547 2024-11-08T10:18:50,648 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=231 (was 219) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50547@0x39cf731a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50547@0x050886eb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50547@0x39cf731a-SendThread(127.0.0.1:50547) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50547@0x050886eb-SendThread(127.0.0.1:50547) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: hconnection-0x2e97f5f2-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50547@0x050886eb java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$737/0x00007eff40b458e0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/2db67accb017:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1354860652_22 at /127.0.0.1:39888 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50547@0x39cf731a java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$737/0x00007eff40b458e0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 333), ProcessCount=11 (was 11), AvailableMemoryMB=2888 (was 2937) 2024-11-08T10:18:50,663 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=231, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=2882 2024-11-08T10:18:50,665 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21d54c3e to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fa88a39 2024-11-08T10:18:50,670 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x21d54c3e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df88b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:50,673 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57d51ee9 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f51372a 2024-11-08T10:18:50,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2041faf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:18:50,687 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:18:50,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T10:18:50,692 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:18:50,692 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:50,693 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-08T10:18:50,695 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:18:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T10:18:50,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741839_1015 (size=423) 2024-11-08T10:18:50,734 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 54d468ed537025de01d7e24f13cf3a2e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:18:50,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741840_1016 (size=74) 2024-11-08T10:18:50,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:50,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 54d468ed537025de01d7e24f13cf3a2e, disabling compactions & flushes 2024-11-08T10:18:50,762 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:50,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:50,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. after waiting 0 ms 2024-11-08T10:18:50,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:50,762 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:50,762 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 54d468ed537025de01d7e24f13cf3a2e: 2024-11-08T10:18:50,765 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:18:50,765 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731061130765"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061130765"}]},"ts":"1731061130765"} 2024-11-08T10:18:50,769 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:18:50,772 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:18:50,772 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061130772"}]},"ts":"1731061130772"} 2024-11-08T10:18:50,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:18:50,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=54d468ed537025de01d7e24f13cf3a2e, ASSIGN}] 2024-11-08T10:18:50,785 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=54d468ed537025de01d7e24f13cf3a2e, ASSIGN 2024-11-08T10:18:50,787 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=54d468ed537025de01d7e24f13cf3a2e, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:18:50,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=54d468ed537025de01d7e24f13cf3a2e, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:50,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 54d468ed537025de01d7e24f13cf3a2e, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:18:51,106 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:18:51,113 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:51,113 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => 54d468ed537025de01d7e24f13cf3a2e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:18:51,114 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,114 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:18:51,114 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for 54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,114 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for 54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,117 INFO [StoreOpener-54d468ed537025de01d7e24f13cf3a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,120 INFO [StoreOpener-54d468ed537025de01d7e24f13cf3a2e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d468ed537025de01d7e24f13cf3a2e columnFamilyName testFamily 2024-11-08T10:18:51,120 DEBUG [StoreOpener-54d468ed537025de01d7e24f13cf3a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:18:51,121 INFO [StoreOpener-54d468ed537025de01d7e24f13cf3a2e-1 {}] regionserver.HStore(327): Store=54d468ed537025de01d7e24f13cf3a2e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:18:51,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,124 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for 54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:18:51,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:18:51,133 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened 54d468ed537025de01d7e24f13cf3a2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70413841, jitterRate=0.049247995018959045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:18:51,134 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for 54d468ed537025de01d7e24f13cf3a2e: 2024-11-08T10:18:51,136 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e., pid=14, masterSystemTime=1731061131106 2024-11-08T10:18:51,140 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:51,140 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:18:51,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=54d468ed537025de01d7e24f13cf3a2e, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:18:51,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T10:18:51,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 54d468ed537025de01d7e24f13cf3a2e, server=2db67accb017,37913,1731061095589 in 202 msec 2024-11-08T10:18:51,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-08T10:18:51,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=54d468ed537025de01d7e24f13cf3a2e, ASSIGN in 370 msec 2024-11-08T10:18:51,157 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:18:51,157 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061131157"}]},"ts":"1731061131157"} 2024-11-08T10:18:51,160 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:18:51,164 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:18:51,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 477 msec 2024-11-08T10:18:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T10:18:51,699 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-08T10:18:51,699 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:18:51,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:18:51,706 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T10:18:51,706 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:18:51,706 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T10:18:51,709 DEBUG [hconnection-0x47a52836-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:18:51,712 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43000, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:18:51,727 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:18:51,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:18:51,734 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:18:53,211 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:18:53,257 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-08T10:18:53,258 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T10:18:53,259 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-08T10:18:53,260 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-08T10:18:55,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T10:18:55,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:18:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T10:18:55,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T10:18:55,230 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T10:18:55,230 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-08T10:18:55,231 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:18:55,231 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T10:18:55,231 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T10:18:55,231 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:18:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T10:18:58,760 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:19:00,741 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11012: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e., hostname=2db67accb017,37913,1731061095589, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11012: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:19:00,742 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T10:19:00.741Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11012: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11012: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:19:00,743 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9010 2024-11-08T10:19:00,743 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57d51ee9 to 127.0.0.1:50547 2024-11-08T10:19:00,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:00,743 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:00,744 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:00,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:00,744 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21d54c3e to 127.0.0.1:50547 2024-11-08T10:19:00,753 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=224 (was 231), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=349 (was 333) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2841 (was 2882) 2024-11-08T10:19:00,761 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=224, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=349, ProcessCount=11, AvailableMemoryMB=2841 2024-11-08T10:19:00,763 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73280625 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42555ef1 2024-11-08T10:19:00,768 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x73280625 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bd8c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:00,769 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ce73a72 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37c242bb 2024-11-08T10:19:00,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e86348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:00,775 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:00,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T10:19:00,778 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:00,778 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:00,778 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-08T10:19:00,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T10:19:00,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741841_1017 (size=421) 2024-11-08T10:19:00,791 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => eb62f731f61df3966926bac4cef0c26e, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:00,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741842_1018 (size=72) 2024-11-08T10:19:00,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:00,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing eb62f731f61df3966926bac4cef0c26e, disabling compactions & flushes 2024-11-08T10:19:00,799 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:00,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:00,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. after waiting 0 ms 2024-11-08T10:19:00,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:00,799 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:00,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for eb62f731f61df3966926bac4cef0c26e: 2024-11-08T10:19:00,801 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:00,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731061140801"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061140801"}]},"ts":"1731061140801"} 2024-11-08T10:19:00,803 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:00,805 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:00,805 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061140805"}]},"ts":"1731061140805"} 2024-11-08T10:19:00,807 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:00,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=eb62f731f61df3966926bac4cef0c26e, ASSIGN}] 2024-11-08T10:19:00,812 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=eb62f731f61df3966926bac4cef0c26e, ASSIGN 2024-11-08T10:19:00,814 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=eb62f731f61df3966926bac4cef0c26e, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:00,964 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=eb62f731f61df3966926bac4cef0c26e, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:00,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure eb62f731f61df3966926bac4cef0c26e, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:01,121 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:01,126 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:01,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => eb62f731f61df3966926bac4cef0c26e, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:01,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:01,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,129 INFO [StoreOpener-eb62f731f61df3966926bac4cef0c26e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,132 INFO [StoreOpener-eb62f731f61df3966926bac4cef0c26e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb62f731f61df3966926bac4cef0c26e columnFamilyName testFamily 2024-11-08T10:19:01,132 DEBUG [StoreOpener-eb62f731f61df3966926bac4cef0c26e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:01,133 INFO [StoreOpener-eb62f731f61df3966926bac4cef0c26e-1 {}] regionserver.HStore(327): Store=eb62f731f61df3966926bac4cef0c26e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:01,134 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,138 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:19:01,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:01,143 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened eb62f731f61df3966926bac4cef0c26e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63962023, jitterRate=-0.04689158499240875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:01,144 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for eb62f731f61df3966926bac4cef0c26e: 2024-11-08T10:19:01,145 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e., pid=17, masterSystemTime=1731061141121 2024-11-08T10:19:01,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:01,149 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:19:01,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=eb62f731f61df3966926bac4cef0c26e, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:01,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-08T10:19:01,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure eb62f731f61df3966926bac4cef0c26e, server=2db67accb017,37913,1731061095589 in 185 msec 2024-11-08T10:19:01,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T10:19:01,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=eb62f731f61df3966926bac4cef0c26e, ASSIGN in 345 msec 2024-11-08T10:19:01,160 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:01,160 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061141160"}]},"ts":"1731061141160"} 2024-11-08T10:19:01,162 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:01,166 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:01,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 391 msec 2024-11-08T10:19:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T10:19:01,783 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-08T10:19:01,783 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:01,783 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:01,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:01,787 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:01,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T10:19:01,789 DEBUG [hconnection-0x2bf41c5a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:01,791 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39192, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:01,802 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:01,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:01,806 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:19:01,808 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x73280625 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:19:01,808 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x73280625 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:01,810 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39208, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:19:01,813 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:19:01,813 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463643 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:01,814 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:19:01,814 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463643 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:01,815 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T10:19:05,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T10:19:05,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:05,261 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:19:05,263 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-08T10:19:06,660 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:19:10,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9007 2024-11-08T10:19:10,821 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ce73a72 to 127.0.0.1:50547 2024-11-08T10:19:10,821 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463643 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:19:10,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:10,822 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:10,822 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:10,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087973' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T10:19:10,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:10,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:39208 deadline: 1731061156821, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087973' does not exist 2024-11-08T10:19:10,822 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x73280625 to 127.0.0.1:50547 2024-11-08T10:19:10,823 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2db67accb017,37913,1731061095589 for closing scanner id = -4907804905177087973 for eb62f731f61df3966926bac4cef0c26e of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T10:19:10,835 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=224 (was 224), OpenFileDescriptor=449 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=319 (was 349), ProcessCount=11 (was 11), AvailableMemoryMB=2827 (was 2841) 2024-11-08T10:19:10,844 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=224, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=319, ProcessCount=11, AvailableMemoryMB=2827 2024-11-08T10:19:10,846 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75423ee5 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7589db70 2024-11-08T10:19:10,850 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x75423ee5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5f94f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:10,851 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fcc7ba4 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@448e4f47 2024-11-08T10:19:10,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20dff425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:10,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T10:19:10,861 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:10,862 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:10,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-08T10:19:10,863 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T10:19:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741843_1019 (size=432) 2024-11-08T10:19:11,278 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 68acff9b3e69cb0077e2babe1fe22e73, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:11,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741844_1020 (size=83) 2024-11-08T10:19:11,288 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:11,288 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 68acff9b3e69cb0077e2babe1fe22e73, disabling compactions & flushes 2024-11-08T10:19:11,289 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,289 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,289 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. after waiting 0 ms 2024-11-08T10:19:11,289 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,289 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,289 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 68acff9b3e69cb0077e2babe1fe22e73: 2024-11-08T10:19:11,291 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:11,291 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731061151291"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061151291"}]},"ts":"1731061151291"} 2024-11-08T10:19:11,294 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:11,296 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:11,296 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061151296"}]},"ts":"1731061151296"} 2024-11-08T10:19:11,298 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:11,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=68acff9b3e69cb0077e2babe1fe22e73, ASSIGN}] 2024-11-08T10:19:11,308 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=68acff9b3e69cb0077e2babe1fe22e73, ASSIGN 2024-11-08T10:19:11,309 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=68acff9b3e69cb0077e2babe1fe22e73, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:11,316 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:19:11,460 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=68acff9b3e69cb0077e2babe1fe22e73, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:11,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 68acff9b3e69cb0077e2babe1fe22e73, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:11,617 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:11,625 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,625 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => 68acff9b3e69cb0077e2babe1fe22e73, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:11,626 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,626 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:11,626 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for 68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,626 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for 68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,628 INFO [StoreOpener-68acff9b3e69cb0077e2babe1fe22e73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,631 INFO [StoreOpener-68acff9b3e69cb0077e2babe1fe22e73-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68acff9b3e69cb0077e2babe1fe22e73 columnFamilyName testFamily 2024-11-08T10:19:11,631 DEBUG [StoreOpener-68acff9b3e69cb0077e2babe1fe22e73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:11,632 INFO [StoreOpener-68acff9b3e69cb0077e2babe1fe22e73-1 {}] regionserver.HStore(327): Store=68acff9b3e69cb0077e2babe1fe22e73/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:11,633 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,634 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,637 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for 68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:19:11,644 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:11,645 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened 68acff9b3e69cb0077e2babe1fe22e73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70220778, jitterRate=0.046371132135391235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:11,646 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for 68acff9b3e69cb0077e2babe1fe22e73: 2024-11-08T10:19:11,647 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73., pid=20, masterSystemTime=1731061151617 2024-11-08T10:19:11,651 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,651 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:19:11,652 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=68acff9b3e69cb0077e2babe1fe22e73, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:11,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T10:19:11,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 68acff9b3e69cb0077e2babe1fe22e73, server=2db67accb017,37913,1731061095589 in 192 msec 2024-11-08T10:19:11,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-08T10:19:11,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=68acff9b3e69cb0077e2babe1fe22e73, ASSIGN in 353 msec 2024-11-08T10:19:11,663 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:11,664 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061151663"}]},"ts":"1731061151663"} 2024-11-08T10:19:11,666 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:11,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:11,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 811 msec 2024-11-08T10:19:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T10:19:11,868 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-08T10:19:11,869 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:11,869 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:11,874 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:11,874 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:11,874 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-08T10:19:11,877 DEBUG [hconnection-0x24a2d1a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:11,884 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58894, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:11,897 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:11,901 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T10:19:11,903 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T10:19:11,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463647 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:11,905 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T10:19:11,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463647 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:11,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087969 2024-11-08T10:19:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:58894 deadline: 1731061152909, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:19:11,910 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:19:11,912 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T10:19:11,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463648 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:11,915 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T10:19:11,915 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:19:11,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:19:11,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463649 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:11,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087967 2024-11-08T10:19:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:58894 deadline: 1731061152918, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:19:11,919 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:19:11,921 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:19:11,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463650 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:11,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-4907804905177087966 2024-11-08T10:19:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:58894 deadline: 1731061152922, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:19:11,923 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:19:11,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:19:11,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463651 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:11,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-4907804905177087965 2024-11-08T10:19:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:58894 deadline: 1731061152926, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:19:11,927 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:19:11,929 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:19:11,930 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3fcc7ba4 to 127.0.0.1:50547 2024-11-08T10:19:11,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:11,930 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:11,930 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:11,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:11,931 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75423ee5 to 127.0.0.1:50547 2024-11-08T10:19:11,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=230 (was 224) - Thread LEAK? -, OpenFileDescriptor=454 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=319 (was 319), ProcessCount=11 (was 11), AvailableMemoryMB=2821 (was 2827) 2024-11-08T10:19:11,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=229, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=319, ProcessCount=11, AvailableMemoryMB=2821 2024-11-08T10:19:11,958 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2545544a to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@494cd1d 2024-11-08T10:19:11,962 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x2545544a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37777a88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:11,964 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7f3e8811 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@73ffd475 2024-11-08T10:19:11,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c973944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:11,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T10:19:11,975 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:11,975 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:11,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-08T10:19:11,976 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T10:19:12,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741845_1021 (size=426) 2024-11-08T10:19:12,003 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d1ef8d97072ec94ee606727c3fa0252e, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:12,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741846_1022 (size=77) 2024-11-08T10:19:12,018 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:12,018 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing d1ef8d97072ec94ee606727c3fa0252e, disabling compactions & flushes 2024-11-08T10:19:12,018 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,018 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,018 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. after waiting 0 ms 2024-11-08T10:19:12,018 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,018 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for d1ef8d97072ec94ee606727c3fa0252e: 2024-11-08T10:19:12,020 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:12,021 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731061152021"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061152021"}]},"ts":"1731061152021"} 2024-11-08T10:19:12,024 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:12,025 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:12,026 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061152025"}]},"ts":"1731061152025"} 2024-11-08T10:19:12,028 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:12,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=d1ef8d97072ec94ee606727c3fa0252e, ASSIGN}] 2024-11-08T10:19:12,034 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=d1ef8d97072ec94ee606727c3fa0252e, ASSIGN 2024-11-08T10:19:12,036 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=d1ef8d97072ec94ee606727c3fa0252e, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:12,187 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=d1ef8d97072ec94ee606727c3fa0252e, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:12,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure d1ef8d97072ec94ee606727c3fa0252e, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:12,349 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:12,355 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,355 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => d1ef8d97072ec94ee606727c3fa0252e, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:12,356 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,357 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:12,357 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,357 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,360 INFO [StoreOpener-d1ef8d97072ec94ee606727c3fa0252e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,371 INFO [StoreOpener-d1ef8d97072ec94ee606727c3fa0252e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1ef8d97072ec94ee606727c3fa0252e columnFamilyName testFamily 2024-11-08T10:19:12,372 DEBUG [StoreOpener-d1ef8d97072ec94ee606727c3fa0252e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:12,373 INFO [StoreOpener-d1ef8d97072ec94ee606727c3fa0252e-1 {}] regionserver.HStore(327): Store=d1ef8d97072ec94ee606727c3fa0252e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:12,375 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,375 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,379 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:19:12,382 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:12,383 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened d1ef8d97072ec94ee606727c3fa0252e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61049361, jitterRate=-0.09029363095760345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:12,384 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for d1ef8d97072ec94ee606727c3fa0252e: 2024-11-08T10:19:12,385 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e., pid=23, masterSystemTime=1731061152348 2024-11-08T10:19:12,388 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,389 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:19:12,389 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=d1ef8d97072ec94ee606727c3fa0252e, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:12,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-08T10:19:12,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure d1ef8d97072ec94ee606727c3fa0252e, server=2db67accb017,37913,1731061095589 in 197 msec 2024-11-08T10:19:12,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T10:19:12,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=d1ef8d97072ec94ee606727c3fa0252e, ASSIGN in 362 msec 2024-11-08T10:19:12,399 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:12,399 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061152399"}]},"ts":"1731061152399"} 2024-11-08T10:19:12,402 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:12,405 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:12,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 433 msec 2024-11-08T10:19:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T10:19:12,979 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-08T10:19:12,979 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:12,980 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:12,985 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:12,985 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:12,986 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T10:19:12,988 DEBUG [hconnection-0x1d75efd8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:12,991 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58906, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:13,004 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:13,007 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:13,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:19:13,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T10:19:13,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:19:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T10:19:15,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:19:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:58906 deadline: 1731061156012, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:19:15,033 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:19:15,033 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463656 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:19:15,035 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2028 2024-11-08T10:19:15,036 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7f3e8811 to 127.0.0.1:50547 2024-11-08T10:19:15,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:15,036 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:15,036 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:15,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:15,036 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2545544a to 127.0.0.1:50547 2024-11-08T10:19:15,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226 (was 229), OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 319) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2815 (was 2821) 2024-11-08T10:19:15,058 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=226, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=2815 2024-11-08T10:19:15,060 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5061e1d5 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32933ee8 2024-11-08T10:19:15,064 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x5061e1d5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae1609a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:15,066 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c059ec4 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68f07fb2 2024-11-08T10:19:15,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7c06a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:15,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T10:19:15,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:15,079 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:15,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-08T10:19:15,081 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T10:19:15,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741847_1023 (size=427) 2024-11-08T10:19:15,092 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ce9524b9626bb8e191e7ad056f5d2da8, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741848_1024 (size=78) 2024-11-08T10:19:15,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:15,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ce9524b9626bb8e191e7ad056f5d2da8, disabling compactions & flushes 2024-11-08T10:19:15,101 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. after waiting 0 ms 2024-11-08T10:19:15,102 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,102 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,102 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ce9524b9626bb8e191e7ad056f5d2da8: 2024-11-08T10:19:15,103 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:15,103 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731061155103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061155103"}]},"ts":"1731061155103"} 2024-11-08T10:19:15,106 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:15,107 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:15,107 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061155107"}]},"ts":"1731061155107"} 2024-11-08T10:19:15,109 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:15,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce9524b9626bb8e191e7ad056f5d2da8, ASSIGN}] 2024-11-08T10:19:15,116 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce9524b9626bb8e191e7ad056f5d2da8, ASSIGN 2024-11-08T10:19:15,118 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce9524b9626bb8e191e7ad056f5d2da8, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:15,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T10:19:15,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:15,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T10:19:15,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:15,268 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=ce9524b9626bb8e191e7ad056f5d2da8, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:15,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure ce9524b9626bb8e191e7ad056f5d2da8, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:15,425 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:15,431 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,431 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => ce9524b9626bb8e191e7ad056f5d2da8, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:15,431 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,431 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:15,431 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,431 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,434 INFO [StoreOpener-ce9524b9626bb8e191e7ad056f5d2da8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,436 INFO [StoreOpener-ce9524b9626bb8e191e7ad056f5d2da8-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce9524b9626bb8e191e7ad056f5d2da8 columnFamilyName testFamily 2024-11-08T10:19:15,436 DEBUG [StoreOpener-ce9524b9626bb8e191e7ad056f5d2da8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:15,437 INFO [StoreOpener-ce9524b9626bb8e191e7ad056f5d2da8-1 {}] regionserver.HStore(327): Store=ce9524b9626bb8e191e7ad056f5d2da8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:15,438 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,439 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,442 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:19:15,446 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:15,447 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened ce9524b9626bb8e191e7ad056f5d2da8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67464718, jitterRate=0.005302637815475464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:15,448 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for ce9524b9626bb8e191e7ad056f5d2da8: 2024-11-08T10:19:15,449 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8., pid=26, masterSystemTime=1731061155425 2024-11-08T10:19:15,452 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,452 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:19:15,453 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=ce9524b9626bb8e191e7ad056f5d2da8, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:15,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T10:19:15,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure ce9524b9626bb8e191e7ad056f5d2da8, server=2db67accb017,37913,1731061095589 in 184 msec 2024-11-08T10:19:15,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T10:19:15,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce9524b9626bb8e191e7ad056f5d2da8, ASSIGN in 344 msec 2024-11-08T10:19:15,463 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:15,463 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061155463"}]},"ts":"1731061155463"} 2024-11-08T10:19:15,465 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:15,469 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:15,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 394 msec 2024-11-08T10:19:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T10:19:16,084 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-08T10:19:16,084 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:16,084 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:16,089 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:16,089 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:16,089 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T10:19:16,092 DEBUG [hconnection-0x585052c8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:16,098 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58912, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:16,109 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:16,113 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:19:16,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:19:17,128 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:19:17,246 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46031,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:19:17,265 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-08T10:19:17,265 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-08T10:19:17,266 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-08T10:19:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T10:19:17,897 ERROR [LeaseRenewer:jenkins@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46031,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:19:18,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:19:19,121 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5017: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8., hostname=2db67accb017,37913,1731061095589, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5017: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:19:19,122 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T10:19:19.122Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5017: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5017: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=993ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:19:19,123 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3009 2024-11-08T10:19:19,124 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c059ec4 to 127.0.0.1:50547 2024-11-08T10:19:19,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:19,124 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:19,124 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:19,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:19,124 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5061e1d5 to 127.0.0.1:50547 2024-11-08T10:19:19,138 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=223 (was 226), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 334) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2807 (was 2815) 2024-11-08T10:19:19,150 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=223, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=2807 2024-11-08T10:19:19,152 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x698c9135 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4aee8f68 2024-11-08T10:19:19,157 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x698c9135 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c5ed26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:19,159 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x408fb736 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a17de58 2024-11-08T10:19:19,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671fef16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:19,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T10:19:19,176 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:19,176 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:19,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-08T10:19:19,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T10:19:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741849_1025 (size=425) 2024-11-08T10:19:19,611 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 645e70ee8fcf32e5b76639c429a17b6d, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:19,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741850_1026 (size=76) 2024-11-08T10:19:19,623 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:19,623 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 645e70ee8fcf32e5b76639c429a17b6d, disabling compactions & flushes 2024-11-08T10:19:19,623 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,623 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,623 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. after waiting 0 ms 2024-11-08T10:19:19,624 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,624 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,624 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 645e70ee8fcf32e5b76639c429a17b6d: 2024-11-08T10:19:19,625 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:19,626 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731061159626"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061159626"}]},"ts":"1731061159626"} 2024-11-08T10:19:19,629 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:19,630 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:19:19,631 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:19,632 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061159631"}]},"ts":"1731061159631"} 2024-11-08T10:19:19,634 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:19,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=645e70ee8fcf32e5b76639c429a17b6d, ASSIGN}] 2024-11-08T10:19:19,640 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=645e70ee8fcf32e5b76639c429a17b6d, ASSIGN 2024-11-08T10:19:19,642 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=645e70ee8fcf32e5b76639c429a17b6d, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:19,793 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=645e70ee8fcf32e5b76639c429a17b6d, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:19,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 645e70ee8fcf32e5b76639c429a17b6d, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:19,949 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:19,955 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,955 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => 645e70ee8fcf32e5b76639c429a17b6d, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:19,956 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,956 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:19,956 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for 645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,956 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for 645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,958 INFO [StoreOpener-645e70ee8fcf32e5b76639c429a17b6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,961 INFO [StoreOpener-645e70ee8fcf32e5b76639c429a17b6d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645e70ee8fcf32e5b76639c429a17b6d columnFamilyName testFamily 2024-11-08T10:19:19,961 DEBUG [StoreOpener-645e70ee8fcf32e5b76639c429a17b6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:19,962 INFO [StoreOpener-645e70ee8fcf32e5b76639c429a17b6d-1 {}] regionserver.HStore(327): Store=645e70ee8fcf32e5b76639c429a17b6d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:19,963 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,964 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,969 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for 645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:19:19,973 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:19,974 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened 645e70ee8fcf32e5b76639c429a17b6d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74928434, jitterRate=0.11652067303657532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:19,975 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for 645e70ee8fcf32e5b76639c429a17b6d: 2024-11-08T10:19:19,976 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d., pid=29, masterSystemTime=1731061159949 2024-11-08T10:19:19,979 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,979 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:19:19,980 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=645e70ee8fcf32e5b76639c429a17b6d, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:19,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T10:19:19,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 645e70ee8fcf32e5b76639c429a17b6d, server=2db67accb017,37913,1731061095589 in 186 msec 2024-11-08T10:19:19,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-08T10:19:19,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=645e70ee8fcf32e5b76639c429a17b6d, ASSIGN in 346 msec 2024-11-08T10:19:19,989 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:19,990 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061159989"}]},"ts":"1731061159989"} 2024-11-08T10:19:19,992 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:19,995 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:19,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 823 msec 2024-11-08T10:19:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T10:19:20,181 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-08T10:19:20,182 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:20,182 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:20,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:20,187 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:20,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T10:19:20,190 DEBUG [hconnection-0x28715adc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:20,192 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49802, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:20,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:20,208 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:20,209 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:19:20,210 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x698c9135 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:19:20,211 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x698c9135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:20,218 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49808, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:19:20,223 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:19:20,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463668 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:20,224 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:19:20,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463668 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:20,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T10:19:23,231 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3006 2024-11-08T10:19:23,231 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x408fb736 to 127.0.0.1:50547 2024-11-08T10:19:23,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:23,231 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:23,231 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463668 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:19:23,231 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:23,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:23,232 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x698c9135 to 127.0.0.1:50547 2024-11-08T10:19:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087948' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T10:19:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:49808 deadline: 1731061164231, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087948' does not exist 2024-11-08T10:19:23,234 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:19:23,232 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2db67accb017,37913,1731061095589 for closing scanner id = -4907804905177087948 for 645e70ee8fcf32e5b76639c429a17b6d of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T10:19:23,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=228 (was 223) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 347) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2796 (was 2807) 2024-11-08T10:19:23,249 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=2796 2024-11-08T10:19:23,251 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43b30ac5 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e2f54ff 2024-11-08T10:19:23,255 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x43b30ac5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73021c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:23,256 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5d9350d9 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a7abfb3 2024-11-08T10:19:23,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c4791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:23,263 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:23,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-08T10:19:23,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:23,266 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:23,266 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-08T10:19:23,267 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:23,268 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-08T10:19:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T10:19:23,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741851_1027 (size=417) 2024-11-08T10:19:23,681 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7574e219918a9b10e7232c85198e870b, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:23,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741852_1028 (size=68) 2024-11-08T10:19:23,725 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:19:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 7574e219918a9b10e7232c85198e870b, disabling compactions & flushes 2024-11-08T10:19:24,090 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. after waiting 0 ms 2024-11-08T10:19:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,090 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,090 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7574e219918a9b10e7232c85198e870b: 2024-11-08T10:19:24,092 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:24,092 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731061164092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061164092"}]},"ts":"1731061164092"} 2024-11-08T10:19:24,095 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:24,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:24,096 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061164096"}]},"ts":"1731061164096"} 2024-11-08T10:19:24,098 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:24,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=7574e219918a9b10e7232c85198e870b, ASSIGN}] 2024-11-08T10:19:24,103 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=7574e219918a9b10e7232c85198e870b, ASSIGN 2024-11-08T10:19:24,103 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=7574e219918a9b10e7232c85198e870b, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:24,254 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=7574e219918a9b10e7232c85198e870b, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:24,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 7574e219918a9b10e7232c85198e870b, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T10:19:24,409 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:24,414 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 7574e219918a9b10e7232c85198e870b, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:24,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:24,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,417 INFO [StoreOpener-7574e219918a9b10e7232c85198e870b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,419 INFO [StoreOpener-7574e219918a9b10e7232c85198e870b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7574e219918a9b10e7232c85198e870b columnFamilyName testFamily 2024-11-08T10:19:24,419 DEBUG [StoreOpener-7574e219918a9b10e7232c85198e870b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:24,420 INFO [StoreOpener-7574e219918a9b10e7232c85198e870b-1 {}] regionserver.HStore(327): Store=7574e219918a9b10e7232c85198e870b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:24,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,426 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 7574e219918a9b10e7232c85198e870b 2024-11-08T10:19:24,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:24,430 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 7574e219918a9b10e7232c85198e870b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65409059, jitterRate=-0.025329068303108215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:24,431 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 7574e219918a9b10e7232c85198e870b: 2024-11-08T10:19:24,432 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b., pid=32, masterSystemTime=1731061164409 2024-11-08T10:19:24,435 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,435 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:19:24,436 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=7574e219918a9b10e7232c85198e870b, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:24,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T10:19:24,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 7574e219918a9b10e7232c85198e870b, server=2db67accb017,37913,1731061095589 in 181 msec 2024-11-08T10:19:24,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T10:19:24,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=7574e219918a9b10e7232c85198e870b, ASSIGN in 339 msec 2024-11-08T10:19:24,444 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:24,444 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061164444"}]},"ts":"1731061164444"} 2024-11-08T10:19:24,446 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:24,449 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:24,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 1.1850 sec 2024-11-08T10:19:25,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:19:25,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T10:19:25,481 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:25,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T10:19:25,481 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:25,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T10:19:25,482 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T10:19:26,272 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-08T10:19:26,272 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:26,272 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:26,276 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:26,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:26,277 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-08T10:19:26,278 DEBUG [hconnection-0x2facd44f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:26,280 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49812, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:26,290 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:26,292 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:26,294 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463672 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:19:26,294 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T10:19:29,249 INFO [master/2db67accb017:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T10:19:29,249 INFO [master/2db67accb017:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T10:19:29,270 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-08T10:19:32,795 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-08T10:19:32,795 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5d9350d9 to 127.0.0.1:50547 2024-11-08T10:19:32,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:32,796 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:32,796 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:32,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:32,796 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x43b30ac5 to 127.0.0.1:50547 2024-11-08T10:19:32,807 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 228), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 351), ProcessCount=11 (was 11), AvailableMemoryMB=2796 (was 2796) 2024-11-08T10:19:32,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=2796 2024-11-08T10:19:32,819 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x142bb6e9 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@77252cc4 2024-11-08T10:19:32,824 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x142bb6e9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6528fb77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:32,826 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29fb98e6 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64c9969f 2024-11-08T10:19:32,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1ce811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:32,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-08T10:19:32,836 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:32,837 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:32,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-08T10:19:32,838 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T10:19:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741853_1029 (size=421) 2024-11-08T10:19:32,853 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 989074082986a01232c976f6de58a84d, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:32,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741854_1030 (size=72) 2024-11-08T10:19:33,270 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:33,271 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 989074082986a01232c976f6de58a84d, disabling compactions & flushes 2024-11-08T10:19:33,271 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,271 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,271 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. after waiting 0 ms 2024-11-08T10:19:33,271 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,271 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,271 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 989074082986a01232c976f6de58a84d: 2024-11-08T10:19:33,273 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:33,273 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731061173273"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061173273"}]},"ts":"1731061173273"} 2024-11-08T10:19:33,278 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:33,279 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:33,279 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061173279"}]},"ts":"1731061173279"} 2024-11-08T10:19:33,281 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:33,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=989074082986a01232c976f6de58a84d, ASSIGN}] 2024-11-08T10:19:33,286 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=989074082986a01232c976f6de58a84d, ASSIGN 2024-11-08T10:19:33,288 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=989074082986a01232c976f6de58a84d, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:33,439 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=989074082986a01232c976f6de58a84d, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:33,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 989074082986a01232c976f6de58a84d, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:33,594 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:33,602 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,602 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => 989074082986a01232c976f6de58a84d, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:33,603 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,603 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:33,603 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for 989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,603 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for 989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,605 INFO [StoreOpener-989074082986a01232c976f6de58a84d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,608 INFO [StoreOpener-989074082986a01232c976f6de58a84d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 989074082986a01232c976f6de58a84d columnFamilyName testFamily 2024-11-08T10:19:33,609 DEBUG [StoreOpener-989074082986a01232c976f6de58a84d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:33,609 INFO [StoreOpener-989074082986a01232c976f6de58a84d-1 {}] regionserver.HStore(327): Store=989074082986a01232c976f6de58a84d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:33,610 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,611 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,614 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for 989074082986a01232c976f6de58a84d 2024-11-08T10:19:33,618 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:33,618 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened 989074082986a01232c976f6de58a84d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59953829, jitterRate=-0.10661832988262177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:33,619 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for 989074082986a01232c976f6de58a84d: 2024-11-08T10:19:33,620 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d., pid=35, masterSystemTime=1731061173594 2024-11-08T10:19:33,623 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,624 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:19:33,624 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=989074082986a01232c976f6de58a84d, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:33,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-08T10:19:33,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 989074082986a01232c976f6de58a84d, server=2db67accb017,37913,1731061095589 in 185 msec 2024-11-08T10:19:33,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T10:19:33,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=989074082986a01232c976f6de58a84d, ASSIGN in 345 msec 2024-11-08T10:19:33,633 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:33,633 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061173633"}]},"ts":"1731061173633"} 2024-11-08T10:19:33,635 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:33,638 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:33,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 804 msec 2024-11-08T10:19:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T10:19:33,843 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-08T10:19:33,843 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:33,843 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:33,848 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:33,849 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:33,849 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-08T10:19:33,851 DEBUG [hconnection-0x648b02ce-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:33,853 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41282, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:33,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:33,871 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:33,879 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463676 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:19:33,880 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T10:19:35,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T10:19:35,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:35,271 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-08T10:19:35,381 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1509 2024-11-08T10:19:35,381 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29fb98e6 to 127.0.0.1:50547 2024-11-08T10:19:35,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:35,382 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:35,382 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:35,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:35,382 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x142bb6e9 to 127.0.0.1:50547 2024-11-08T10:19:35,393 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 323), ProcessCount=11 (was 11), AvailableMemoryMB=2787 (was 2796) 2024-11-08T10:19:35,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=2787 2024-11-08T10:19:35,407 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c5a1b82 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67778567 2024-11-08T10:19:35,411 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x5c5a1b82 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334510ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:35,412 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49992fc7 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5edd56e0 2024-11-08T10:19:35,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d7c0b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:35,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-08T10:19:35,422 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:35,423 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:35,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-08T10:19:35,424 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T10:19:35,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741855_1031 (size=420) 2024-11-08T10:19:35,841 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 182d6d30146365e85f2f0b8f61460763, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:35,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741856_1032 (size=71) 2024-11-08T10:19:35,857 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:35,857 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 182d6d30146365e85f2f0b8f61460763, disabling compactions & flushes 2024-11-08T10:19:35,857 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:35,857 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:35,857 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. after waiting 0 ms 2024-11-08T10:19:35,857 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:35,857 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:35,857 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 182d6d30146365e85f2f0b8f61460763: 2024-11-08T10:19:35,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:35,859 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731061175859"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061175859"}]},"ts":"1731061175859"} 2024-11-08T10:19:35,863 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:35,864 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:35,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061175864"}]},"ts":"1731061175864"} 2024-11-08T10:19:35,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:35,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=182d6d30146365e85f2f0b8f61460763, ASSIGN}] 2024-11-08T10:19:35,875 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=182d6d30146365e85f2f0b8f61460763, ASSIGN 2024-11-08T10:19:35,876 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=182d6d30146365e85f2f0b8f61460763, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:36,027 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=182d6d30146365e85f2f0b8f61460763, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:36,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 182d6d30146365e85f2f0b8f61460763, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:36,182 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:36,196 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:36,196 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 182d6d30146365e85f2f0b8f61460763, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:36,196 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,196 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:36,197 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,197 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,199 INFO [StoreOpener-182d6d30146365e85f2f0b8f61460763-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,201 INFO [StoreOpener-182d6d30146365e85f2f0b8f61460763-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 182d6d30146365e85f2f0b8f61460763 columnFamilyName testFamily 2024-11-08T10:19:36,201 DEBUG [StoreOpener-182d6d30146365e85f2f0b8f61460763-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:36,202 INFO [StoreOpener-182d6d30146365e85f2f0b8f61460763-1 {}] regionserver.HStore(327): Store=182d6d30146365e85f2f0b8f61460763/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:36,203 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,204 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,207 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 182d6d30146365e85f2f0b8f61460763 2024-11-08T10:19:36,212 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:36,213 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 182d6d30146365e85f2f0b8f61460763; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67738231, jitterRate=0.009378299117088318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:36,214 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 182d6d30146365e85f2f0b8f61460763: 2024-11-08T10:19:36,215 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763., pid=38, masterSystemTime=1731061176182 2024-11-08T10:19:36,217 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:36,217 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:19:36,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=182d6d30146365e85f2f0b8f61460763, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:36,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-08T10:19:36,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 182d6d30146365e85f2f0b8f61460763, server=2db67accb017,37913,1731061095589 in 190 msec 2024-11-08T10:19:36,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T10:19:36,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=182d6d30146365e85f2f0b8f61460763, ASSIGN in 348 msec 2024-11-08T10:19:36,226 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:36,226 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061176226"}]},"ts":"1731061176226"} 2024-11-08T10:19:36,228 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:36,231 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:36,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 811 msec 2024-11-08T10:19:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T10:19:36,428 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-08T10:19:36,428 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:36,428 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:36,435 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:36,435 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:36,435 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-08T10:19:36,438 DEBUG [hconnection-0x51576207-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:36,441 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41298, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:36,455 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:36,460 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76dd219f to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60f1ce6c 2024-11-08T10:19:36,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7484ae45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:36,465 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:36,472 DEBUG [hconnection-0x57c792a5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:36,476 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41302, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:36,481 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:19:36,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463681 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:36,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:19:36,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463681 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:19:36,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T10:19:37,491 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T10:19:37.491Z, RpcRetryingCaller{globalStartTime=2024-11-08T10:19:36.484Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T10:19:37.491Z, RpcRetryingCaller{globalStartTime=2024-11-08T10:19:36.484Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:19:37,492 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1007 2024-11-08T10:19:37,492 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76dd219f to 127.0.0.1:50547 2024-11-08T10:19:37,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:37,493 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x49992fc7 to 127.0.0.1:50547 2024-11-08T10:19:37,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:37,493 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:37,493 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:37,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:37,493 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c5a1b82 to 127.0.0.1:50547 2024-11-08T10:19:37,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 231) - Thread LEAK? -, OpenFileDescriptor=459 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 321), ProcessCount=11 (was 11), AvailableMemoryMB=2782 (was 2787) 2024-11-08T10:19:37,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=2782 2024-11-08T10:19:37,515 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17f135ef to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a899989 2024-11-08T10:19:37,519 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x17f135ef {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@381f296e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:37,520 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66a3fab5 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a495255 2024-11-08T10:19:37,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148950be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:37,527 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:37,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T10:19:37,530 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:37,530 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:37,530 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-08T10:19:37,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T10:19:37,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741857_1033 (size=422) 2024-11-08T10:19:37,948 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b0d2d9924440ee776e54c19a9c601b36, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:37,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741858_1034 (size=73) 2024-11-08T10:19:37,986 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:19:38,361 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:38,361 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing b0d2d9924440ee776e54c19a9c601b36, disabling compactions & flushes 2024-11-08T10:19:38,361 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,362 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,362 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. after waiting 0 ms 2024-11-08T10:19:38,362 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,362 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,362 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for b0d2d9924440ee776e54c19a9c601b36: 2024-11-08T10:19:38,363 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:38,364 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731061178364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061178364"}]},"ts":"1731061178364"} 2024-11-08T10:19:38,367 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:38,368 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:38,368 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061178368"}]},"ts":"1731061178368"} 2024-11-08T10:19:38,370 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:38,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=b0d2d9924440ee776e54c19a9c601b36, ASSIGN}] 2024-11-08T10:19:38,377 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=b0d2d9924440ee776e54c19a9c601b36, ASSIGN 2024-11-08T10:19:38,378 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=b0d2d9924440ee776e54c19a9c601b36, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:38,529 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=b0d2d9924440ee776e54c19a9c601b36, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:38,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure b0d2d9924440ee776e54c19a9c601b36, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T10:19:38,685 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:38,691 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => b0d2d9924440ee776e54c19a9c601b36, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:38,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:38,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,693 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,697 INFO [StoreOpener-b0d2d9924440ee776e54c19a9c601b36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,700 INFO [StoreOpener-b0d2d9924440ee776e54c19a9c601b36-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0d2d9924440ee776e54c19a9c601b36 columnFamilyName testFamily 2024-11-08T10:19:38,700 DEBUG [StoreOpener-b0d2d9924440ee776e54c19a9c601b36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:38,701 INFO [StoreOpener-b0d2d9924440ee776e54c19a9c601b36-1 {}] regionserver.HStore(327): Store=b0d2d9924440ee776e54c19a9c601b36/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:38,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,705 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:19:38,708 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:38,709 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened b0d2d9924440ee776e54c19a9c601b36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60266820, jitterRate=-0.1019544005393982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:38,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for b0d2d9924440ee776e54c19a9c601b36: 2024-11-08T10:19:38,712 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36., pid=41, masterSystemTime=1731061178685 2024-11-08T10:19:38,715 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,715 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:19:38,715 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=b0d2d9924440ee776e54c19a9c601b36, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:38,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T10:19:38,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure b0d2d9924440ee776e54c19a9c601b36, server=2db67accb017,37913,1731061095589 in 186 msec 2024-11-08T10:19:38,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-08T10:19:38,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=b0d2d9924440ee776e54c19a9c601b36, ASSIGN in 347 msec 2024-11-08T10:19:38,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:38,726 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061178726"}]},"ts":"1731061178726"} 2024-11-08T10:19:38,728 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:38,734 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:38,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.2080 sec 2024-11-08T10:19:39,145 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:19:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T10:19:40,537 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-08T10:19:40,537 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:40,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:40,543 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:40,543 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:40,543 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T10:19:40,546 DEBUG [hconnection-0x659c7779-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:40,547 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34362, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:40,566 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:40,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:19:40,575 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:19:40,576 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T10:19:41,273 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-08T10:19:41,274 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-08T10:19:43,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:19:44,195 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:19:45,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T10:19:45,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:45,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T10:19:45,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:47,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T10:19:47,587 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:19:47,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:34362 deadline: 1731061189576, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:19:47,588 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:19:47,588 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463685 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:19:47,589 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7016 2024-11-08T10:19:47,589 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66a3fab5 to 127.0.0.1:50547 2024-11-08T10:19:47,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:47,589 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:47,590 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:47,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:47,590 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17f135ef to 127.0.0.1:50547 2024-11-08T10:19:47,600 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=231 (was 236), OpenFileDescriptor=454 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 321) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2762 (was 2782) 2024-11-08T10:19:47,609 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=2762 2024-11-08T10:19:47,611 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4819798c to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@466b9adc 2024-11-08T10:19:47,614 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x4819798c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13393da9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:47,616 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ca87f42 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28dcb000 2024-11-08T10:19:47,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0a698e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:47,622 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:47,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T10:19:47,625 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:47,625 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:47,625 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-08T10:19:47,626 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T10:19:47,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741859_1035 (size=428) 2024-11-08T10:19:48,038 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1ba716bec7899a38c7fd594140143301, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:48,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741860_1036 (size=79) 2024-11-08T10:19:48,408 DEBUG [master/2db67accb017:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T10:19:48,413 DEBUG [master/2db67accb017:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region c98f706b3d1970465cfb9fe302b92eeb changed from -1.0 to 0.0, refreshing cache 2024-11-08T10:19:48,458 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:48,458 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 1ba716bec7899a38c7fd594140143301, disabling compactions & flushes 2024-11-08T10:19:48,458 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,458 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,458 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. after waiting 0 ms 2024-11-08T10:19:48,458 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,458 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,458 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1ba716bec7899a38c7fd594140143301: 2024-11-08T10:19:48,459 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:48,460 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731061188460"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061188460"}]},"ts":"1731061188460"} 2024-11-08T10:19:48,462 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:48,463 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:48,464 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061188463"}]},"ts":"1731061188463"} 2024-11-08T10:19:48,465 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:48,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1ba716bec7899a38c7fd594140143301, ASSIGN}] 2024-11-08T10:19:48,471 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1ba716bec7899a38c7fd594140143301, ASSIGN 2024-11-08T10:19:48,472 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1ba716bec7899a38c7fd594140143301, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:48,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=1ba716bec7899a38c7fd594140143301, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:48,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 1ba716bec7899a38c7fd594140143301, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:19:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T10:19:48,779 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:19:48,786 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 1ba716bec7899a38c7fd594140143301, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:19:48,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:48,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,790 INFO [StoreOpener-1ba716bec7899a38c7fd594140143301-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,793 INFO [StoreOpener-1ba716bec7899a38c7fd594140143301-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ba716bec7899a38c7fd594140143301 columnFamilyName testFamily 2024-11-08T10:19:48,793 DEBUG [StoreOpener-1ba716bec7899a38c7fd594140143301-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:48,794 INFO [StoreOpener-1ba716bec7899a38c7fd594140143301-1 {}] regionserver.HStore(327): Store=1ba716bec7899a38c7fd594140143301/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:19:48,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 1ba716bec7899a38c7fd594140143301 2024-11-08T10:19:48,804 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:19:48,805 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 1ba716bec7899a38c7fd594140143301; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72232112, jitterRate=0.07634234428405762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:19:48,807 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 1ba716bec7899a38c7fd594140143301: 2024-11-08T10:19:48,808 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., pid=44, masterSystemTime=1731061188779 2024-11-08T10:19:48,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,810 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:19:48,811 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=1ba716bec7899a38c7fd594140143301, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:48,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-08T10:19:48,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 1ba716bec7899a38c7fd594140143301, server=2db67accb017,37913,1731061095589 in 187 msec 2024-11-08T10:19:48,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T10:19:48,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=1ba716bec7899a38c7fd594140143301, ASSIGN in 346 msec 2024-11-08T10:19:48,819 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:19:48,819 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061188819"}]},"ts":"1731061188819"} 2024-11-08T10:19:48,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:19:48,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:19:48,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.2020 sec 2024-11-08T10:19:49,191 ERROR [LeaseRenewer:jenkins@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46031,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:19:49,975 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46031,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:19:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T10:19:50,631 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-08T10:19:50,631 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:19:50,631 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:50,636 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T10:19:50,636 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:19:50,636 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T10:19:50,638 DEBUG [hconnection-0x6ddcaf22-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:50,640 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60256, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:50,652 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:19:50,671 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:19:50,671 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:19:50,673 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x4819798c {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:19:50,673 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x4819798c {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:19:50,674 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60266, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:19:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:19:50,677 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:19:53,276 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T10:19:54,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:19:55,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T10:19:55,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:19:56,682 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-08T10:19:56,683 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-08T10:19:56,683 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:19:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T10:19:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:19:57,692 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:19:58,309 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087980 expired clientIPAndPort=172.17.0.2:43000, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:19:59,681 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms 2024-11-08T10:19:59,681 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms 2024-11-08T10:19:59,681 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:19:59,681 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T10:19:56.683Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-08T10:19:59.681Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T10:19:59,683 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9012 2024-11-08T10:19:59,684 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2ca87f42 to 127.0.0.1:50547 2024-11-08T10:19:59,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:59,684 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:19:59,684 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:19:59,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:19:59,684 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4819798c to 127.0.0.1:50547 2024-11-08T10:19:59,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=226 (was 231), OpenFileDescriptor=444 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 371), ProcessCount=11 (was 11), AvailableMemoryMB=2759 (was 2762) 2024-11-08T10:19:59,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=2758 2024-11-08T10:19:59,710 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x72d3bf34 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@126feca5 2024-11-08T10:19:59,714 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x72d3bf34 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60860b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:59,716 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a6b9908 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60ff76f9 2024-11-08T10:19:59,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fba5fcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:19:59,723 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:19:59,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-08T10:19:59,726 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:19:59,726 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:19:59,726 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-08T10:19:59,727 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:19:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T10:19:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741861_1037 (size=416) 2024-11-08T10:19:59,745 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 00032c2457ddc4b4cee4c234010c2624, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:19:59,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741862_1038 (size=67) 2024-11-08T10:19:59,753 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:19:59,753 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 00032c2457ddc4b4cee4c234010c2624, disabling compactions & flushes 2024-11-08T10:19:59,753 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:19:59,753 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:19:59,753 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. after waiting 0 ms 2024-11-08T10:19:59,753 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:19:59,753 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:19:59,753 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 00032c2457ddc4b4cee4c234010c2624: 2024-11-08T10:19:59,755 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:19:59,755 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731061199755"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061199755"}]},"ts":"1731061199755"} 2024-11-08T10:19:59,757 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:19:59,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:19:59,758 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061199758"}]},"ts":"1731061199758"} 2024-11-08T10:19:59,760 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:19:59,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=00032c2457ddc4b4cee4c234010c2624, ASSIGN}] 2024-11-08T10:19:59,765 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=00032c2457ddc4b4cee4c234010c2624, ASSIGN 2024-11-08T10:19:59,767 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=00032c2457ddc4b4cee4c234010c2624, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:19:59,917 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=00032c2457ddc4b4cee4c234010c2624, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:19:59,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 00032c2457ddc4b4cee4c234010c2624, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:00,074 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:00,083 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:20:00,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 00032c2457ddc4b4cee4c234010c2624, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:00,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:00,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,087 INFO [StoreOpener-00032c2457ddc4b4cee4c234010c2624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,089 INFO [StoreOpener-00032c2457ddc4b4cee4c234010c2624-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00032c2457ddc4b4cee4c234010c2624 columnFamilyName testFamily 2024-11-08T10:20:00,089 DEBUG [StoreOpener-00032c2457ddc4b4cee4c234010c2624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:00,090 INFO [StoreOpener-00032c2457ddc4b4cee4c234010c2624-1 {}] regionserver.HStore(327): Store=00032c2457ddc4b4cee4c234010c2624/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:00,092 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,109 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:20:00,113 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:00,114 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 00032c2457ddc4b4cee4c234010c2624; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72482460, jitterRate=0.08007282018661499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:00,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 00032c2457ddc4b4cee4c234010c2624: 2024-11-08T10:20:00,117 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624., pid=47, masterSystemTime=1731061200073 2024-11-08T10:20:00,119 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:20:00,119 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:20:00,120 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=00032c2457ddc4b4cee4c234010c2624, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:00,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T10:20:00,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 00032c2457ddc4b4cee4c234010c2624, server=2db67accb017,37913,1731061095589 in 202 msec 2024-11-08T10:20:00,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-08T10:20:00,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=00032c2457ddc4b4cee4c234010c2624, ASSIGN in 360 msec 2024-11-08T10:20:00,127 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:00,128 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061200127"}]},"ts":"1731061200127"} 2024-11-08T10:20:00,130 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:20:00,133 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:00,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 410 msec 2024-11-08T10:20:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T10:20:00,731 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-08T10:20:00,731 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:20:00,731 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:00,736 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T10:20:00,736 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:00,736 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-08T10:20:00,740 DEBUG [hconnection-0x12827016-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:00,742 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57456, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:00,760 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:00,782 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66eec451 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68f5c368 2024-11-08T10:20:00,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a1c923f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:00,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:20:00,812 DEBUG [hconnection-0x4c34ee97-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:00,814 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57462, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:00,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:20:00,828 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463698 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:00,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:20:00,831 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463698 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:00,831 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T10:20:04,192 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:05,316 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087978 expired clientIPAndPort=172.17.0.2:43000, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:20:05,616 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:05,662 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T10:20:05,662 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:05,664 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-08T10:20:06,841 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T10:20:06.841Z, RpcRetryingCaller{globalStartTime=2024-11-08T10:20:00.830Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T10:20:06.841Z, RpcRetryingCaller{globalStartTime=2024-11-08T10:20:00.830Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:20:06,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6011 2024-11-08T10:20:06,841 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66eec451 to 127.0.0.1:50547 2024-11-08T10:20:06,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:06,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a6b9908 to 127.0.0.1:50547 2024-11-08T10:20:06,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:06,842 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:06,842 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:06,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:06,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x72d3bf34 to 127.0.0.1:50547 2024-11-08T10:20:06,856 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=235 (was 226) - Thread LEAK? -, OpenFileDescriptor=451 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 330) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2701 (was 2758) 2024-11-08T10:20:06,866 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=235, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=2701 2024-11-08T10:20:06,868 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b54e756 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57a401f6 2024-11-08T10:20:06,872 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x1b54e756 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4457f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:06,873 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34fc2d74 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65d82942 2024-11-08T10:20:06,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598a0cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:06,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T10:20:06,882 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:06,882 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:06,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-08T10:20:06,884 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T10:20:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741863_1039 (size=432) 2024-11-08T10:20:06,901 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9a51f8d80633f3cc5b5a69d57da95629, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:06,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741864_1040 (size=83) 2024-11-08T10:20:06,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:06,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 9a51f8d80633f3cc5b5a69d57da95629, disabling compactions & flushes 2024-11-08T10:20:06,916 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:06,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:06,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. after waiting 0 ms 2024-11-08T10:20:06,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:06,916 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:06,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9a51f8d80633f3cc5b5a69d57da95629: 2024-11-08T10:20:06,917 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:06,917 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731061206917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061206917"}]},"ts":"1731061206917"} 2024-11-08T10:20:06,920 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:06,921 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:06,921 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061206921"}]},"ts":"1731061206921"} 2024-11-08T10:20:06,922 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T10:20:06,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=9a51f8d80633f3cc5b5a69d57da95629, ASSIGN}] 2024-11-08T10:20:06,928 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=9a51f8d80633f3cc5b5a69d57da95629, ASSIGN 2024-11-08T10:20:06,929 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=9a51f8d80633f3cc5b5a69d57da95629, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:07,080 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=9a51f8d80633f3cc5b5a69d57da95629, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:07,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 9a51f8d80633f3cc5b5a69d57da95629, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:07,234 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:07,240 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:07,240 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => 9a51f8d80633f3cc5b5a69d57da95629, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:07,241 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,241 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:07,241 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for 9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,241 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for 9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,243 INFO [StoreOpener-9a51f8d80633f3cc5b5a69d57da95629-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,246 INFO [StoreOpener-9a51f8d80633f3cc5b5a69d57da95629-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a51f8d80633f3cc5b5a69d57da95629 columnFamilyName testFamily 2024-11-08T10:20:07,246 DEBUG [StoreOpener-9a51f8d80633f3cc5b5a69d57da95629-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:07,247 INFO [StoreOpener-9a51f8d80633f3cc5b5a69d57da95629-1 {}] regionserver.HStore(327): Store=9a51f8d80633f3cc5b5a69d57da95629/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:07,248 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,248 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,251 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for 9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:20:07,256 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:07,257 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened 9a51f8d80633f3cc5b5a69d57da95629; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63596350, jitterRate=-0.052340537309646606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:07,258 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for 9a51f8d80633f3cc5b5a69d57da95629: 2024-11-08T10:20:07,259 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., pid=50, masterSystemTime=1731061207234 2024-11-08T10:20:07,262 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=9a51f8d80633f3cc5b5a69d57da95629, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:07,263 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:07,264 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:20:07,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-08T10:20:07,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 9a51f8d80633f3cc5b5a69d57da95629, server=2db67accb017,37913,1731061095589 in 183 msec 2024-11-08T10:20:07,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T10:20:07,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=9a51f8d80633f3cc5b5a69d57da95629, ASSIGN in 342 msec 2024-11-08T10:20:07,272 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:07,272 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061207272"}]},"ts":"1731061207272"} 2024-11-08T10:20:07,274 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T10:20:07,277 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:07,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 397 msec 2024-11-08T10:20:07,332 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T10:20:07,887 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-08T10:20:07,887 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T10:20:07,887 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:07,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T10:20:07,892 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:07,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T10:20:07,894 DEBUG [hconnection-0x56c68c64-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:07,896 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57474, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:07,906 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:07,909 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:20:07,910 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:20:07,911 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x1b54e756 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:20:07,911 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x1b54e756 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:07,913 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57478, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:20:07,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:20:08,921 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-08T10:20:08,921 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-08T10:20:08,921 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:20:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T10:20:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:20:09,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:20:10,911 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=978ms 2024-11-08T10:20:10,911 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=978ms 2024-11-08T10:20:10,912 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:20:10,912 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T10:20:08.921Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-08T10:20:10.912Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=978ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T10:20:10,914 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2024-11-08T10:20:10,914 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34fc2d74 to 127.0.0.1:50547 2024-11-08T10:20:10,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:10,914 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:10,914 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:10,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:10,915 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b54e756 to 127.0.0.1:50547 2024-11-08T10:20:10,926 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=232 (was 235), OpenFileDescriptor=450 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=357 (was 336) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2691 (was 2701) 2024-11-08T10:20:10,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=357, ProcessCount=11, AvailableMemoryMB=2690 2024-11-08T10:20:10,938 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7c6344d2 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2697f6f2 2024-11-08T10:20:10,942 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x7c6344d2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1620a921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:10,943 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31604b76 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f6c2f46 2024-11-08T10:20:10,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cfb1ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:10,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T10:20:10,956 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:10,957 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:10,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-08T10:20:10,958 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T10:20:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741865_1041 (size=436) 2024-11-08T10:20:11,323 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087973 expired clientIPAndPort=172.17.0.2:39208, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:20:11,378 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8ba0428adc53ccbb2911449553de4eb7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:11,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741866_1042 (size=87) 2024-11-08T10:20:11,392 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:11,392 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 8ba0428adc53ccbb2911449553de4eb7, disabling compactions & flushes 2024-11-08T10:20:11,392 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,392 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,392 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. after waiting 0 ms 2024-11-08T10:20:11,392 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,392 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,392 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8ba0428adc53ccbb2911449553de4eb7: 2024-11-08T10:20:11,394 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:11,394 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731061211394"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061211394"}]},"ts":"1731061211394"} 2024-11-08T10:20:11,396 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:11,397 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:11,398 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061211397"}]},"ts":"1731061211397"} 2024-11-08T10:20:11,400 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:11,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=8ba0428adc53ccbb2911449553de4eb7, ASSIGN}] 2024-11-08T10:20:11,411 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=8ba0428adc53ccbb2911449553de4eb7, ASSIGN 2024-11-08T10:20:11,411 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=8ba0428adc53ccbb2911449553de4eb7, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:11,433 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:11,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=8ba0428adc53ccbb2911449553de4eb7, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:11,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 8ba0428adc53ccbb2911449553de4eb7, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:11,666 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T10:20:11,719 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:11,725 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,725 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => 8ba0428adc53ccbb2911449553de4eb7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:11,726 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,726 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:11,726 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for 8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,726 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for 8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,728 INFO [StoreOpener-8ba0428adc53ccbb2911449553de4eb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,730 INFO [StoreOpener-8ba0428adc53ccbb2911449553de4eb7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ba0428adc53ccbb2911449553de4eb7 columnFamilyName testFamily 2024-11-08T10:20:11,730 DEBUG [StoreOpener-8ba0428adc53ccbb2911449553de4eb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:11,731 INFO [StoreOpener-8ba0428adc53ccbb2911449553de4eb7-1 {}] regionserver.HStore(327): Store=8ba0428adc53ccbb2911449553de4eb7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:11,732 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,733 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,735 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for 8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:20:11,746 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:11,747 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened 8ba0428adc53ccbb2911449553de4eb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64224177, jitterRate=-0.042985185980796814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:11,748 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for 8ba0428adc53ccbb2911449553de4eb7: 2024-11-08T10:20:11,749 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7., pid=53, masterSystemTime=1731061211719 2024-11-08T10:20:11,753 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=8ba0428adc53ccbb2911449553de4eb7, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:11,753 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,753 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:20:11,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T10:20:11,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 8ba0428adc53ccbb2911449553de4eb7, server=2db67accb017,37913,1731061095589 in 188 msec 2024-11-08T10:20:11,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-08T10:20:11,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=8ba0428adc53ccbb2911449553de4eb7, ASSIGN in 349 msec 2024-11-08T10:20:11,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:11,763 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061211762"}]},"ts":"1731061211762"} 2024-11-08T10:20:11,764 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:11,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:11,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 814 msec 2024-11-08T10:20:11,923 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087966 expired clientIPAndPort=172.17.0.2:58894, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:20:11,924 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087967 expired clientIPAndPort=172.17.0.2:58894, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:20:11,924 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087969 expired clientIPAndPort=172.17.0.2:58894, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:20:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T10:20:11,963 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-08T10:20:11,964 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:11,964 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:11,968 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:11,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:11,969 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-08T10:20:11,971 DEBUG [hconnection-0x49d0b56f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:11,973 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59728, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:11,983 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:11,986 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T10:20:11,987 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:20:11,988 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x7c6344d2 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:20:11,988 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x7c6344d2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:11,990 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59738, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:20:11,993 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T10:20:11,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463710 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:11,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T10:20:11,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463710 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:11,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087906 2024-11-08T10:20:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59738 deadline: 1731061214993, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:11,996 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463710 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:11,997 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T10:20:11,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463711 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:11,998 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T10:20:11,998 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:11,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:11,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463712 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:11,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087904 2024-11-08T10:20:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59738 deadline: 1731061214998, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:12,000 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463712 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:12,002 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:12,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463713 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:12,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-4907804905177087903 2024-11-08T10:20:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59738 deadline: 1731061215001, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:12,003 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463713 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:12,004 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:12,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463714 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:12,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-4907804905177087902 2024-11-08T10:20:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59738 deadline: 1731061215003, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:12,005 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463714 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:12,006 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:12,006 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31604b76 to 127.0.0.1:50547 2024-11-08T10:20:12,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:12,007 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:12,007 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:12,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:12,007 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7c6344d2 to 127.0.0.1:50547 2024-11-08T10:20:12,018 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=233 (was 232) - Thread LEAK? -, OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=357 (was 357), ProcessCount=11 (was 11), AvailableMemoryMB=2674 (was 2690) 2024-11-08T10:20:12,028 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=357, ProcessCount=11, AvailableMemoryMB=2674 2024-11-08T10:20:12,030 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e1cb6bc to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5454ecbf 2024-11-08T10:20:12,034 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x2e1cb6bc {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4779b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:12,035 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29d58c6b to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@734860b 2024-11-08T10:20:12,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c566907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:12,042 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:12,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T10:20:12,044 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:12,045 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:12,045 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-08T10:20:12,045 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T10:20:12,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741867_1043 (size=422) 2024-11-08T10:20:12,062 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 75db9bee13abc31f9df9651eb584b046, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:12,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741868_1044 (size=73) 2024-11-08T10:20:12,076 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:12,076 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 75db9bee13abc31f9df9651eb584b046, disabling compactions & flushes 2024-11-08T10:20:12,076 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,076 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,076 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. after waiting 0 ms 2024-11-08T10:20:12,076 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,076 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,076 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 75db9bee13abc31f9df9651eb584b046: 2024-11-08T10:20:12,077 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:12,078 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731061212077"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061212077"}]},"ts":"1731061212077"} 2024-11-08T10:20:12,080 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:12,081 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:12,081 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061212081"}]},"ts":"1731061212081"} 2024-11-08T10:20:12,083 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:12,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=75db9bee13abc31f9df9651eb584b046, ASSIGN}] 2024-11-08T10:20:12,089 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=75db9bee13abc31f9df9651eb584b046, ASSIGN 2024-11-08T10:20:12,090 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=75db9bee13abc31f9df9651eb584b046, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:12,241 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=75db9bee13abc31f9df9651eb584b046, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:12,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 75db9bee13abc31f9df9651eb584b046, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:12,396 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:12,402 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,402 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => 75db9bee13abc31f9df9651eb584b046, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:12,402 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,402 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:12,403 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for 75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,403 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for 75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,407 INFO [StoreOpener-75db9bee13abc31f9df9651eb584b046-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,409 INFO [StoreOpener-75db9bee13abc31f9df9651eb584b046-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75db9bee13abc31f9df9651eb584b046 columnFamilyName testFamily 2024-11-08T10:20:12,409 DEBUG [StoreOpener-75db9bee13abc31f9df9651eb584b046-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:12,410 INFO [StoreOpener-75db9bee13abc31f9df9651eb584b046-1 {}] regionserver.HStore(327): Store=75db9bee13abc31f9df9651eb584b046/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:12,412 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,419 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for 75db9bee13abc31f9df9651eb584b046 2024-11-08T10:20:12,425 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:12,426 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened 75db9bee13abc31f9df9651eb584b046; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70391414, jitterRate=0.048913806676864624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:12,427 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for 75db9bee13abc31f9df9651eb584b046: 2024-11-08T10:20:12,429 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046., pid=56, masterSystemTime=1731061212396 2024-11-08T10:20:12,432 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,432 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:20:12,432 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=75db9bee13abc31f9df9651eb584b046, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:12,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T10:20:12,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 75db9bee13abc31f9df9651eb584b046, server=2db67accb017,37913,1731061095589 in 190 msec 2024-11-08T10:20:12,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T10:20:12,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=75db9bee13abc31f9df9651eb584b046, ASSIGN in 348 msec 2024-11-08T10:20:12,439 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:12,439 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061212439"}]},"ts":"1731061212439"} 2024-11-08T10:20:12,441 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:12,444 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:12,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 402 msec 2024-11-08T10:20:12,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T10:20:13,049 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-08T10:20:13,049 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:13,049 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:13,053 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:13,053 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:13,053 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T10:20:13,055 DEBUG [hconnection-0x6661a18c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:13,056 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59740, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:13,065 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:13,067 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:20:13,068 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:20:13,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:20:15,127 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087960 expired clientIPAndPort=172.17.0.2:58906, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:20:15,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T10:20:15,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:15,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T10:20:15,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:15,230 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T10:20:15,230 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:17,629 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087955 expired clientIPAndPort=172.17.0.2:58912, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:20:17,905 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:17,960 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-08T10:20:17,961 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-08T10:20:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T10:20:19,631 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087953 expired clientIPAndPort=172.17.0.2:58912, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:20:20,083 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:20:22,072 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046., hostname=2db67accb017,37913,1731061095589, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:20:22,072 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T10:20:22.072Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:20:22,073 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2024-11-08T10:20:22,073 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29d58c6b to 127.0.0.1:50547 2024-11-08T10:20:22,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:22,073 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:22,074 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:22,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:22,074 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e1cb6bc to 127.0.0.1:50547 2024-11-08T10:20:22,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=229 (was 233), OpenFileDescriptor=448 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=396 (was 357) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2649 (was 2674) 2024-11-08T10:20:22,091 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=2649 2024-11-08T10:20:22,092 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3600c963 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d6dbddf 2024-11-08T10:20:22,095 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x3600c963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f70d2a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:22,096 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b0c80a2 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42d7a33 2024-11-08T10:20:22,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3456f004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:22,102 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:22,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T10:20:22,104 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:22,104 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:22,105 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-08T10:20:22,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T10:20:22,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741869_1045 (size=420) 2024-11-08T10:20:22,516 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4e72ba4990ef0072afcae1a4ad9ca190, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:22,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741870_1046 (size=71) 2024-11-08T10:20:22,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:22,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 4e72ba4990ef0072afcae1a4ad9ca190, disabling compactions & flushes 2024-11-08T10:20:22,924 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:22,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:22,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. after waiting 0 ms 2024-11-08T10:20:22,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:22,924 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:22,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4e72ba4990ef0072afcae1a4ad9ca190: 2024-11-08T10:20:22,925 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:22,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731061222926"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061222926"}]},"ts":"1731061222926"} 2024-11-08T10:20:22,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:22,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:22,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061222929"}]},"ts":"1731061222929"} 2024-11-08T10:20:22,930 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:22,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=4e72ba4990ef0072afcae1a4ad9ca190, ASSIGN}] 2024-11-08T10:20:22,935 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=4e72ba4990ef0072afcae1a4ad9ca190, ASSIGN 2024-11-08T10:20:22,936 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=4e72ba4990ef0072afcae1a4ad9ca190, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:23,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=4e72ba4990ef0072afcae1a4ad9ca190, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:23,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 4e72ba4990ef0072afcae1a4ad9ca190, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T10:20:23,241 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:23,247 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:23,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => 4e72ba4990ef0072afcae1a4ad9ca190, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:23,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:23,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for 4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for 4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,249 INFO [StoreOpener-4e72ba4990ef0072afcae1a4ad9ca190-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,251 INFO [StoreOpener-4e72ba4990ef0072afcae1a4ad9ca190-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e72ba4990ef0072afcae1a4ad9ca190 columnFamilyName testFamily 2024-11-08T10:20:23,251 DEBUG [StoreOpener-4e72ba4990ef0072afcae1a4ad9ca190-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:23,252 INFO [StoreOpener-4e72ba4990ef0072afcae1a4ad9ca190-1 {}] regionserver.HStore(327): Store=4e72ba4990ef0072afcae1a4ad9ca190/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:23,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,255 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for 4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:20:23,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:23,258 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened 4e72ba4990ef0072afcae1a4ad9ca190; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68837586, jitterRate=0.025759965181350708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:23,259 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for 4e72ba4990ef0072afcae1a4ad9ca190: 2024-11-08T10:20:23,260 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190., pid=59, masterSystemTime=1731061223241 2024-11-08T10:20:23,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:23,262 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:20:23,262 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=4e72ba4990ef0072afcae1a4ad9ca190, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:23,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T10:20:23,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 4e72ba4990ef0072afcae1a4ad9ca190, server=2db67accb017,37913,1731061095589 in 175 msec 2024-11-08T10:20:23,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T10:20:23,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=4e72ba4990ef0072afcae1a4ad9ca190, ASSIGN in 331 msec 2024-11-08T10:20:23,268 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:23,268 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061223268"}]},"ts":"1731061223268"} 2024-11-08T10:20:23,269 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:23,272 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:23,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 1.1700 sec 2024-11-08T10:20:23,734 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087948 expired clientIPAndPort=172.17.0.2:49808, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:20:23,962 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-08T10:20:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T10:20:25,109 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-08T10:20:25,109 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:25,109 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:25,113 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:25,113 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:25,113 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T10:20:25,115 DEBUG [hconnection-0x3775b041-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:25,117 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49688, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:25,124 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:25,126 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:20:25,126 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:20:25,127 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x3600c963 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:20:25,128 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x3600c963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:25,129 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49690, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:20:25,131 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:20:25,131 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463726 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:25,132 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:20:25,132 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463726 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:25,133 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T10:20:25,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T10:20:25,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:26,584 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:28,790 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:29,737 ERROR [LeaseRenewer:jenkins@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46031,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:20:30,115 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46031,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:20:34,141 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9009 2024-11-08T10:20:34,141 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b0c80a2 to 127.0.0.1:50547 2024-11-08T10:20:34,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:34,142 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:34,142 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463726 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:34,142 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:34,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:34,142 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3600c963 to 127.0.0.1:50547 2024-11-08T10:20:34,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087890' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T10:20:34,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:49690 deadline: 1731061240141, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087890' does not exist 2024-11-08T10:20:34,145 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:34,142 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2db67accb017,37913,1731061095589 for closing scanner id = -4907804905177087890 for 4e72ba4990ef0072afcae1a4ad9ca190 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T10:20:34,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227 (was 229), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 396), ProcessCount=11 (was 11), AvailableMemoryMB=3473 (was 2649) - AvailableMemoryMB LEAK? - 2024-11-08T10:20:34,161 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=329, ProcessCount=11, AvailableMemoryMB=3472 2024-11-08T10:20:34,163 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x79c4fd3d to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3669e927 2024-11-08T10:20:34,166 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x79c4fd3d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63256ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:34,167 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fe540c9 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@19b14f98 2024-11-08T10:20:34,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47427486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:34,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T10:20:34,176 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:34,176 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:34,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-08T10:20:34,176 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T10:20:34,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741871_1047 (size=431) 2024-11-08T10:20:34,592 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5c9bae98c605ef30875f4e0115c93556, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:34,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741872_1048 (size=82) 2024-11-08T10:20:34,633 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:35,000 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:35,000 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 5c9bae98c605ef30875f4e0115c93556, disabling compactions & flushes 2024-11-08T10:20:35,000 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,000 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,000 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. after waiting 0 ms 2024-11-08T10:20:35,000 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,000 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,000 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5c9bae98c605ef30875f4e0115c93556: 2024-11-08T10:20:35,001 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:35,002 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731061235001"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061235001"}]},"ts":"1731061235001"} 2024-11-08T10:20:35,003 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:35,004 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:35,004 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061235004"}]},"ts":"1731061235004"} 2024-11-08T10:20:35,005 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:35,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=5c9bae98c605ef30875f4e0115c93556, ASSIGN}] 2024-11-08T10:20:35,010 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=5c9bae98c605ef30875f4e0115c93556, ASSIGN 2024-11-08T10:20:35,011 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=5c9bae98c605ef30875f4e0115c93556, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:35,161 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=5c9bae98c605ef30875f4e0115c93556, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:35,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 5c9bae98c605ef30875f4e0115c93556, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T10:20:35,315 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:35,322 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,322 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 5c9bae98c605ef30875f4e0115c93556, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:35,322 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,322 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:35,322 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,322 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,324 INFO [StoreOpener-5c9bae98c605ef30875f4e0115c93556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,326 INFO [StoreOpener-5c9bae98c605ef30875f4e0115c93556-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c9bae98c605ef30875f4e0115c93556 columnFamilyName testFamily 2024-11-08T10:20:35,326 DEBUG [StoreOpener-5c9bae98c605ef30875f4e0115c93556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:35,327 INFO [StoreOpener-5c9bae98c605ef30875f4e0115c93556-1 {}] regionserver.HStore(327): Store=5c9bae98c605ef30875f4e0115c93556/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:35,328 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,328 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,331 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:20:35,335 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:35,336 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 5c9bae98c605ef30875f4e0115c93556; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61339439, jitterRate=-0.08597113192081451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:35,337 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 5c9bae98c605ef30875f4e0115c93556: 2024-11-08T10:20:35,337 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556., pid=62, masterSystemTime=1731061235315 2024-11-08T10:20:35,339 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,339 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:20:35,340 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=5c9bae98c605ef30875f4e0115c93556, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:35,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-08T10:20:35,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 5c9bae98c605ef30875f4e0115c93556, server=2db67accb017,37913,1731061095589 in 177 msec 2024-11-08T10:20:35,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T10:20:35,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=5c9bae98c605ef30875f4e0115c93556, ASSIGN in 334 msec 2024-11-08T10:20:35,346 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:35,346 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061235346"}]},"ts":"1731061235346"} 2024-11-08T10:20:35,348 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:35,350 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:35,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1760 sec 2024-11-08T10:20:35,964 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-08T10:20:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T10:20:37,181 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-08T10:20:37,181 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:37,181 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:37,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:37,185 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:37,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-08T10:20:37,187 DEBUG [hconnection-0x4c219324-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:37,188 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44088, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:37,196 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:37,198 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T10:20:37,199 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T10:20:37,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463730 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:37,200 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T10:20:37,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463730 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:37,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087886 2024-11-08T10:20:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44088 deadline: 1731061240200, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:37,201 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:20:37,203 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T10:20:37,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463731 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:37,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T10:20:37,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:37,206 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:37,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463732 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:37,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-4907804905177087884 2024-11-08T10:20:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44088 deadline: 1731061240206, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:37,207 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:20:37,209 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:37,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463733 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:37,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-4907804905177087883 2024-11-08T10:20:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44088 deadline: 1731061240209, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:37,210 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:20:37,212 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:37,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463734 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:37,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-4907804905177087882 2024-11-08T10:20:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:44088 deadline: 1731061240212, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T10:20:37,213 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:20:37,215 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T10:20:37,215 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1fe540c9 to 127.0.0.1:50547 2024-11-08T10:20:37,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:37,215 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:37,215 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:37,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:37,216 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x79c4fd3d to 127.0.0.1:50547 2024-11-08T10:20:37,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=235 (was 227) - Thread LEAK? -, OpenFileDescriptor=450 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 329), ProcessCount=11 (was 11), AvailableMemoryMB=3364 (was 3472) 2024-11-08T10:20:37,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=235, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=329, ProcessCount=11, AvailableMemoryMB=3364 2024-11-08T10:20:37,237 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2322fc4c to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16a1dcc3 2024-11-08T10:20:37,240 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x2322fc4c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635e9b39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:37,242 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7aa9c35e to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b4c7b3a 2024-11-08T10:20:37,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb83683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:37,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T10:20:37,249 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:37,249 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:37,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-08T10:20:37,250 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T10:20:37,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741873_1049 (size=425) 2024-11-08T10:20:37,660 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f6d402c0ed49fa60db33e36ec6f7dc26, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741874_1050 (size=76) 2024-11-08T10:20:38,048 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087935 expired clientIPAndPort=172.17.0.2:41302, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:20:38,068 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:38,068 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f6d402c0ed49fa60db33e36ec6f7dc26, disabling compactions & flushes 2024-11-08T10:20:38,068 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,068 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,068 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. after waiting 0 ms 2024-11-08T10:20:38,068 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,068 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,068 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f6d402c0ed49fa60db33e36ec6f7dc26: 2024-11-08T10:20:38,069 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:38,069 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731061238069"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061238069"}]},"ts":"1731061238069"} 2024-11-08T10:20:38,071 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:38,071 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:38,072 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061238072"}]},"ts":"1731061238072"} 2024-11-08T10:20:38,073 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:38,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=f6d402c0ed49fa60db33e36ec6f7dc26, ASSIGN}] 2024-11-08T10:20:38,078 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=f6d402c0ed49fa60db33e36ec6f7dc26, ASSIGN 2024-11-08T10:20:38,078 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=f6d402c0ed49fa60db33e36ec6f7dc26, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:38,229 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=f6d402c0ed49fa60db33e36ec6f7dc26, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:38,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure f6d402c0ed49fa60db33e36ec6f7dc26, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T10:20:38,383 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:38,388 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,389 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => f6d402c0ed49fa60db33e36ec6f7dc26, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:38,389 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,389 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:38,389 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,390 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,392 INFO [StoreOpener-f6d402c0ed49fa60db33e36ec6f7dc26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,394 INFO [StoreOpener-f6d402c0ed49fa60db33e36ec6f7dc26-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6d402c0ed49fa60db33e36ec6f7dc26 columnFamilyName testFamily 2024-11-08T10:20:38,394 DEBUG [StoreOpener-f6d402c0ed49fa60db33e36ec6f7dc26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:38,394 INFO [StoreOpener-f6d402c0ed49fa60db33e36ec6f7dc26-1 {}] regionserver.HStore(327): Store=f6d402c0ed49fa60db33e36ec6f7dc26/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:38,395 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,396 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,399 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:20:38,402 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:38,403 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened f6d402c0ed49fa60db33e36ec6f7dc26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67981783, jitterRate=0.013007506728172302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:38,404 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for f6d402c0ed49fa60db33e36ec6f7dc26: 2024-11-08T10:20:38,405 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26., pid=65, masterSystemTime=1731061238383 2024-11-08T10:20:38,407 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,407 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:20:38,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=f6d402c0ed49fa60db33e36ec6f7dc26, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:38,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T10:20:38,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure f6d402c0ed49fa60db33e36ec6f7dc26, server=2db67accb017,37913,1731061095589 in 178 msec 2024-11-08T10:20:38,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-08T10:20:38,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=f6d402c0ed49fa60db33e36ec6f7dc26, ASSIGN in 334 msec 2024-11-08T10:20:38,413 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:38,414 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061238413"}]},"ts":"1731061238413"} 2024-11-08T10:20:38,415 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:38,418 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:38,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.1710 sec 2024-11-08T10:20:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T10:20:40,254 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-08T10:20:40,254 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:40,254 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:40,258 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:40,258 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:40,258 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T10:20:40,260 DEBUG [hconnection-0x56f27c1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:40,262 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46122, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:40,270 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:40,272 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:20:40,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:20:40,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T10:20:40,858 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T10:20:41,966 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-08T10:20:42,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:20:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46122 deadline: 1731061243273, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:20:42,291 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:20:42,292 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463739 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:20:42,293 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2020 2024-11-08T10:20:42,293 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7aa9c35e to 127.0.0.1:50547 2024-11-08T10:20:42,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:42,293 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:42,293 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:42,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:42,293 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2322fc4c to 127.0.0.1:50547 2024-11-08T10:20:42,303 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=235 (was 235), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 329), ProcessCount=11 (was 11), AvailableMemoryMB=3271 (was 3364) 2024-11-08T10:20:42,311 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=235, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=3271 2024-11-08T10:20:42,312 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7709a56f to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30777824 2024-11-08T10:20:42,315 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x7709a56f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:42,316 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4385daa9 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a92950d 2024-11-08T10:20:42,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208a69ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:42,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T10:20:42,323 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:42,324 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:42,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-08T10:20:42,324 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T10:20:42,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741875_1051 (size=426) 2024-11-08T10:20:42,734 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 43a73611ad06dc7aebd9e66c4bb58558, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:42,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741876_1052 (size=77) 2024-11-08T10:20:43,141 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:43,141 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 43a73611ad06dc7aebd9e66c4bb58558, disabling compactions & flushes 2024-11-08T10:20:43,142 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. after waiting 0 ms 2024-11-08T10:20:43,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,142 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 43a73611ad06dc7aebd9e66c4bb58558: 2024-11-08T10:20:43,143 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:43,143 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731061243143"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061243143"}]},"ts":"1731061243143"} 2024-11-08T10:20:43,145 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:43,146 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:43,146 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061243146"}]},"ts":"1731061243146"} 2024-11-08T10:20:43,147 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:43,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=43a73611ad06dc7aebd9e66c4bb58558, ASSIGN}] 2024-11-08T10:20:43,151 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=43a73611ad06dc7aebd9e66c4bb58558, ASSIGN 2024-11-08T10:20:43,152 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=43a73611ad06dc7aebd9e66c4bb58558, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:43,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=43a73611ad06dc7aebd9e66c4bb58558, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:43,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 43a73611ad06dc7aebd9e66c4bb58558, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T10:20:43,457 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:43,462 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,462 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 43a73611ad06dc7aebd9e66c4bb58558, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:43,462 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,463 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:43,463 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,463 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,464 INFO [StoreOpener-43a73611ad06dc7aebd9e66c4bb58558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,466 INFO [StoreOpener-43a73611ad06dc7aebd9e66c4bb58558-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43a73611ad06dc7aebd9e66c4bb58558 columnFamilyName testFamily 2024-11-08T10:20:43,466 DEBUG [StoreOpener-43a73611ad06dc7aebd9e66c4bb58558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:43,467 INFO [StoreOpener-43a73611ad06dc7aebd9e66c4bb58558-1 {}] regionserver.HStore(327): Store=43a73611ad06dc7aebd9e66c4bb58558/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:43,467 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,468 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,470 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:20:43,472 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:43,472 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 43a73611ad06dc7aebd9e66c4bb58558; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74890176, jitterRate=0.1159505844116211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:43,473 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 43a73611ad06dc7aebd9e66c4bb58558: 2024-11-08T10:20:43,474 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558., pid=68, masterSystemTime=1731061243457 2024-11-08T10:20:43,476 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,476 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:20:43,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=43a73611ad06dc7aebd9e66c4bb58558, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:43,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-08T10:20:43,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 43a73611ad06dc7aebd9e66c4bb58558, server=2db67accb017,37913,1731061095589 in 173 msec 2024-11-08T10:20:43,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T10:20:43,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=43a73611ad06dc7aebd9e66c4bb58558, ASSIGN in 329 msec 2024-11-08T10:20:43,481 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:43,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061243481"}]},"ts":"1731061243481"} 2024-11-08T10:20:43,483 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:43,485 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:43,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1630 sec 2024-11-08T10:20:43,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:20:45,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T10:20:45,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:45,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T10:20:45,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:45,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T10:20:45,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T10:20:45,328 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-08T10:20:45,328 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:45,328 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:45,331 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:45,332 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:45,332 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T10:20:45,334 DEBUG [hconnection-0x3b885617-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:45,335 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46124, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:45,342 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:45,344 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:20:45,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:20:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-08T10:20:47,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:20:47,656 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087931 expired clientIPAndPort=172.17.0.2:34362, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:20:47,967 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-08T10:20:48,350 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558., hostname=2db67accb017,37913,1731061095589, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:20:48,351 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T10:20:48.351Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5021: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=987ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:20:48,352 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3007 2024-11-08T10:20:48,352 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4385daa9 to 127.0.0.1:50547 2024-11-08T10:20:48,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:48,352 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:48,352 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:48,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:48,352 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7709a56f to 127.0.0.1:50547 2024-11-08T10:20:48,361 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=231 (was 235), OpenFileDescriptor=451 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 311), ProcessCount=11 (was 11), AvailableMemoryMB=3249 (was 3271) 2024-11-08T10:20:48,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=3249 2024-11-08T10:20:48,371 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x546e28b3 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d89115d 2024-11-08T10:20:48,374 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x546e28b3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a18740, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:48,375 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f828dbc to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44b7f1d2 2024-11-08T10:20:48,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61396247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:48,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T10:20:48,381 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:48,381 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:48,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-08T10:20:48,382 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T10:20:48,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741877_1053 (size=424) 2024-11-08T10:20:48,791 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ec472e3a7c3c4010c8c2b8708ba386f2, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:48,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741878_1054 (size=75) 2024-11-08T10:20:48,864 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:48,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:49,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:49,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing ec472e3a7c3c4010c8c2b8708ba386f2, disabling compactions & flushes 2024-11-08T10:20:49,199 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. after waiting 0 ms 2024-11-08T10:20:49,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,199 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for ec472e3a7c3c4010c8c2b8708ba386f2: 2024-11-08T10:20:49,200 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:49,201 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731061249200"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061249200"}]},"ts":"1731061249200"} 2024-11-08T10:20:49,202 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:49,203 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:49,203 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061249203"}]},"ts":"1731061249203"} 2024-11-08T10:20:49,204 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:49,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=ec472e3a7c3c4010c8c2b8708ba386f2, ASSIGN}] 2024-11-08T10:20:49,214 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=ec472e3a7c3c4010c8c2b8708ba386f2, ASSIGN 2024-11-08T10:20:49,214 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=ec472e3a7c3c4010c8c2b8708ba386f2, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:49,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=ec472e3a7c3c4010c8c2b8708ba386f2, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:49,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure ec472e3a7c3c4010c8c2b8708ba386f2, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T10:20:49,519 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:49,523 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,524 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => ec472e3a7c3c4010c8c2b8708ba386f2, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:49,524 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,524 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:49,524 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,524 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,526 INFO [StoreOpener-ec472e3a7c3c4010c8c2b8708ba386f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,527 INFO [StoreOpener-ec472e3a7c3c4010c8c2b8708ba386f2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec472e3a7c3c4010c8c2b8708ba386f2 columnFamilyName testFamily 2024-11-08T10:20:49,527 DEBUG [StoreOpener-ec472e3a7c3c4010c8c2b8708ba386f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:49,528 INFO [StoreOpener-ec472e3a7c3c4010c8c2b8708ba386f2-1 {}] regionserver.HStore(327): Store=ec472e3a7c3c4010c8c2b8708ba386f2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:49,529 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,529 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,531 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:20:49,533 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:49,534 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened ec472e3a7c3c4010c8c2b8708ba386f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72527338, jitterRate=0.08074155449867249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:49,535 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for ec472e3a7c3c4010c8c2b8708ba386f2: 2024-11-08T10:20:49,536 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2., pid=71, masterSystemTime=1731061249519 2024-11-08T10:20:49,537 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,537 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:20:49,538 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=ec472e3a7c3c4010c8c2b8708ba386f2, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:49,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T10:20:49,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure ec472e3a7c3c4010c8c2b8708ba386f2, server=2db67accb017,37913,1731061095589 in 172 msec 2024-11-08T10:20:49,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-08T10:20:49,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=ec472e3a7c3c4010c8c2b8708ba386f2, ASSIGN in 327 msec 2024-11-08T10:20:49,542 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:49,543 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061249542"}]},"ts":"1731061249542"} 2024-11-08T10:20:49,544 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:49,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:49,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1660 sec 2024-11-08T10:20:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T10:20:51,385 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-08T10:20:51,385 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:51,386 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:51,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:51,389 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:51,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T10:20:51,391 DEBUG [hconnection-0x2e08345c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:51,392 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52910, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:51,399 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:51,401 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:20:51,401 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:20:51,402 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x546e28b3 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:20:51,402 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x546e28b3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:51,403 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52916, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:20:51,405 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:20:51,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463751 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:51,406 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:20:51,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463751 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:51,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T10:20:53,969 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-08T10:20:54,411 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3005 2024-11-08T10:20:54,412 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f828dbc to 127.0.0.1:50547 2024-11-08T10:20:54,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:54,412 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:20:54,412 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463751 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:54,412 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:20:54,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:20:54,412 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x546e28b3 to 127.0.0.1:50547 2024-11-08T10:20:54,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087865' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T10:20:54,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:52916 deadline: 1731061255411, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4907804905177087865' does not exist 2024-11-08T10:20:54,415 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:54,412 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2db67accb017,37913,1731061095589 for closing scanner id = -4907804905177087865 for ec472e3a7c3c4010c8c2b8708ba386f2 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T10:20:54,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231 (was 231), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=3227 (was 3249) 2024-11-08T10:20:54,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=3227 2024-11-08T10:20:54,431 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x212bda2e to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4bb7dae7 2024-11-08T10:20:54,434 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x212bda2e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3a1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:54,435 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d015bb8 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28951e17 2024-11-08T10:20:54,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57904b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:20:54,440 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:20:54,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-08T10:20:54,441 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:20:54,442 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:54,442 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-08T10:20:54,442 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:20:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T10:20:54,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741879_1055 (size=416) 2024-11-08T10:20:54,852 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 18632d949262e24bebe96198d33fa2ce, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:20:54,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741880_1056 (size=67) 2024-11-08T10:20:54,907 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:20:55,026 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:20:55,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T10:20:55,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:20:55,260 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:55,260 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 18632d949262e24bebe96198d33fa2ce, disabling compactions & flushes 2024-11-08T10:20:55,260 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,260 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,260 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. after waiting 0 ms 2024-11-08T10:20:55,260 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,260 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,260 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 18632d949262e24bebe96198d33fa2ce: 2024-11-08T10:20:55,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:20:55,261 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731061255261"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061255261"}]},"ts":"1731061255261"} 2024-11-08T10:20:55,263 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:20:55,264 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:20:55,264 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061255264"}]},"ts":"1731061255264"} 2024-11-08T10:20:55,265 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:20:55,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=18632d949262e24bebe96198d33fa2ce, ASSIGN}] 2024-11-08T10:20:55,270 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=18632d949262e24bebe96198d33fa2ce, ASSIGN 2024-11-08T10:20:55,270 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=18632d949262e24bebe96198d33fa2ce, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:20:55,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=18632d949262e24bebe96198d33fa2ce, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:55,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 18632d949262e24bebe96198d33fa2ce, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:20:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T10:20:55,574 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:20:55,579 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 18632d949262e24bebe96198d33fa2ce, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:20:55,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:20:55,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,581 INFO [StoreOpener-18632d949262e24bebe96198d33fa2ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,583 INFO [StoreOpener-18632d949262e24bebe96198d33fa2ce-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18632d949262e24bebe96198d33fa2ce columnFamilyName testFamily 2024-11-08T10:20:55,583 DEBUG [StoreOpener-18632d949262e24bebe96198d33fa2ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:20:55,583 INFO [StoreOpener-18632d949262e24bebe96198d33fa2ce-1 {}] regionserver.HStore(327): Store=18632d949262e24bebe96198d33fa2ce/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:20:55,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,587 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 18632d949262e24bebe96198d33fa2ce 2024-11-08T10:20:55,590 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:20:55,590 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 18632d949262e24bebe96198d33fa2ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71023001, jitterRate=0.058325186371803284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:20:55,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 18632d949262e24bebe96198d33fa2ce: 2024-11-08T10:20:55,592 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce., pid=74, masterSystemTime=1731061255574 2024-11-08T10:20:55,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,594 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:20:55,594 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=18632d949262e24bebe96198d33fa2ce, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:20:55,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-08T10:20:55,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 18632d949262e24bebe96198d33fa2ce, server=2db67accb017,37913,1731061095589 in 173 msec 2024-11-08T10:20:55,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T10:20:55,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=18632d949262e24bebe96198d33fa2ce, ASSIGN in 328 msec 2024-11-08T10:20:55,598 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:20:55,598 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061255598"}]},"ts":"1731061255598"} 2024-11-08T10:20:55,600 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:20:55,603 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:20:55,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1630 sec 2024-11-08T10:20:57,264 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087925 expired clientIPAndPort=172.17.0.2:60266, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:20:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T10:20:57,446 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-08T10:20:57,446 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:20:57,446 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:57,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-08T10:20:57,452 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463752 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:20:57,453 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T10:20:57,454 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:20:57,454 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-08T10:20:57,456 DEBUG [hconnection-0x1cd03367-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:20:57,457 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52918, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:20:57,464 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:20:57,466 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:20:57,467 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463755 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:20:57,467 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T10:20:59,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-08T10:21:01,081 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:21:03,968 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-08T10:21:03,968 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d015bb8 to 127.0.0.1:50547 2024-11-08T10:21:03,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:03,969 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:03,969 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:03,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:03,969 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x212bda2e to 127.0.0.1:50547 2024-11-08T10:21:03,978 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231 (was 231), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=3179 (was 3227) 2024-11-08T10:21:03,986 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=3178 2024-11-08T10:21:03,988 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1341e2e9 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39035b34 2024-11-08T10:21:03,991 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x1341e2e9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f92b701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:03,991 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7905d42f to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@62d35011 2024-11-08T10:21:03,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e19a008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:03,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:21:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-08T10:21:03,998 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:21:03,998 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:03,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-08T10:21:03,999 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:21:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T10:21:04,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741881_1057 (size=420) 2024-11-08T10:21:04,183 ERROR [LeaseRenewer:jenkins@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46031,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:21:04,272 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087923 expired clientIPAndPort=172.17.0.2:60266, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:21:04,408 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7a9a8e7ae601fb76ff6bcacfb7f43b8c, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:21:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741882_1058 (size=71) 2024-11-08T10:21:04,816 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:04,816 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 7a9a8e7ae601fb76ff6bcacfb7f43b8c, disabling compactions & flushes 2024-11-08T10:21:04,816 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:04,816 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:04,816 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. after waiting 0 ms 2024-11-08T10:21:04,816 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:04,816 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:04,816 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7a9a8e7ae601fb76ff6bcacfb7f43b8c: 2024-11-08T10:21:04,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:21:04,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731061264818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061264818"}]},"ts":"1731061264818"} 2024-11-08T10:21:04,820 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:21:04,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:21:04,821 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061264820"}]},"ts":"1731061264820"} 2024-11-08T10:21:04,822 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:21:04,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a9a8e7ae601fb76ff6bcacfb7f43b8c, ASSIGN}] 2024-11-08T10:21:04,826 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a9a8e7ae601fb76ff6bcacfb7f43b8c, ASSIGN 2024-11-08T10:21:04,827 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a9a8e7ae601fb76ff6bcacfb7f43b8c, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:21:04,977 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=7a9a8e7ae601fb76ff6bcacfb7f43b8c, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:04,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 7a9a8e7ae601fb76ff6bcacfb7f43b8c, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:21:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T10:21:05,131 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:21:05,136 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:05,136 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => 7a9a8e7ae601fb76ff6bcacfb7f43b8c, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:21:05,136 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,136 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:05,136 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for 7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,136 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for 7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,138 INFO [StoreOpener-7a9a8e7ae601fb76ff6bcacfb7f43b8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,139 INFO [StoreOpener-7a9a8e7ae601fb76ff6bcacfb7f43b8c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a9a8e7ae601fb76ff6bcacfb7f43b8c columnFamilyName testFamily 2024-11-08T10:21:05,140 DEBUG [StoreOpener-7a9a8e7ae601fb76ff6bcacfb7f43b8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:05,140 INFO [StoreOpener-7a9a8e7ae601fb76ff6bcacfb7f43b8c-1 {}] regionserver.HStore(327): Store=7a9a8e7ae601fb76ff6bcacfb7f43b8c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:21:05,141 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,141 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,143 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for 7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:05,145 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:21:05,146 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened 7a9a8e7ae601fb76ff6bcacfb7f43b8c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70209458, jitterRate=0.04620245099067688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:21:05,147 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for 7a9a8e7ae601fb76ff6bcacfb7f43b8c: 2024-11-08T10:21:05,147 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c., pid=77, masterSystemTime=1731061265131 2024-11-08T10:21:05,149 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:05,149 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:05,149 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=7a9a8e7ae601fb76ff6bcacfb7f43b8c, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:05,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T10:21:05,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 7a9a8e7ae601fb76ff6bcacfb7f43b8c, server=2db67accb017,37913,1731061095589 in 172 msec 2024-11-08T10:21:05,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-08T10:21:05,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a9a8e7ae601fb76ff6bcacfb7f43b8c, ASSIGN in 327 msec 2024-11-08T10:21:05,154 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:21:05,154 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061265154"}]},"ts":"1731061265154"} 2024-11-08T10:21:05,156 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:21:05,158 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:21:05,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1610 sec 2024-11-08T10:21:05,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T10:21:05,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:05,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T10:21:05,229 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:05,337 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46031,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:21:05,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-08T10:21:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T10:21:07,002 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-08T10:21:07,002 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:21:07,002 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:07,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-08T10:21:07,007 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463756 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:07,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T10:21:07,008 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:07,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-08T10:21:07,010 DEBUG [hconnection-0x18db5697-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:07,011 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46624, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:07,018 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:21:07,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:21:07,021 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463759 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T10:21:07,021 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T10:21:07,375 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087918 expired clientIPAndPort=172.17.0.2:57462, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:21:08,522 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-08T10:21:08,522 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7905d42f to 127.0.0.1:50547 2024-11-08T10:21:08,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:08,522 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:08,522 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:08,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:08,522 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1341e2e9 to 127.0.0.1:50547 2024-11-08T10:21:08,532 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=203 (was 221), ProcessCount=11 (was 11), AvailableMemoryMB=3167 (was 3178) 2024-11-08T10:21:08,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=203, ProcessCount=11, AvailableMemoryMB=3167 2024-11-08T10:21:08,541 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b578082 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52c43c29 2024-11-08T10:21:08,545 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x6b578082 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4639e210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:08,545 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dc94bc0 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2aaf642e 2024-11-08T10:21:08,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7daba269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:08,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:21:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-08T10:21:08,552 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:21:08,552 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:08,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-08T10:21:08,553 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:21:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T10:21:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741883_1059 (size=419) 2024-11-08T10:21:08,963 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4161700af8f79081e6d959b624f78cc1, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:21:08,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741884_1060 (size=70) 2024-11-08T10:21:09,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:09,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 4161700af8f79081e6d959b624f78cc1, disabling compactions & flushes 2024-11-08T10:21:09,371 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. after waiting 0 ms 2024-11-08T10:21:09,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,371 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4161700af8f79081e6d959b624f78cc1: 2024-11-08T10:21:09,372 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:21:09,373 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731061269372"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061269372"}]},"ts":"1731061269372"} 2024-11-08T10:21:09,374 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:21:09,375 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:21:09,375 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061269375"}]},"ts":"1731061269375"} 2024-11-08T10:21:09,376 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:21:09,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=4161700af8f79081e6d959b624f78cc1, ASSIGN}] 2024-11-08T10:21:09,380 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=4161700af8f79081e6d959b624f78cc1, ASSIGN 2024-11-08T10:21:09,381 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=4161700af8f79081e6d959b624f78cc1, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:21:09,476 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087913 expired clientIPAndPort=172.17.0.2:57478, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:09,531 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=4161700af8f79081e6d959b624f78cc1, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:09,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 4161700af8f79081e6d959b624f78cc1, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:21:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T10:21:09,685 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:21:09,690 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,690 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => 4161700af8f79081e6d959b624f78cc1, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:21:09,690 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,690 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:09,690 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for 4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,690 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for 4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,692 INFO [StoreOpener-4161700af8f79081e6d959b624f78cc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,693 INFO [StoreOpener-4161700af8f79081e6d959b624f78cc1-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4161700af8f79081e6d959b624f78cc1 columnFamilyName testFamily 2024-11-08T10:21:09,694 DEBUG [StoreOpener-4161700af8f79081e6d959b624f78cc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:09,694 INFO [StoreOpener-4161700af8f79081e6d959b624f78cc1-1 {}] regionserver.HStore(327): Store=4161700af8f79081e6d959b624f78cc1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:21:09,695 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,695 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,697 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for 4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:09,700 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:21:09,700 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened 4161700af8f79081e6d959b624f78cc1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73830593, jitterRate=0.10016156733036041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:21:09,701 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for 4161700af8f79081e6d959b624f78cc1: 2024-11-08T10:21:09,702 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1., pid=80, masterSystemTime=1731061269685 2024-11-08T10:21:09,703 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,703 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:09,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=4161700af8f79081e6d959b624f78cc1, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:09,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-08T10:21:09,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 4161700af8f79081e6d959b624f78cc1, server=2db67accb017,37913,1731061095589 in 172 msec 2024-11-08T10:21:09,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T10:21:09,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=4161700af8f79081e6d959b624f78cc1, ASSIGN in 326 msec 2024-11-08T10:21:09,708 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:21:09,709 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061269708"}]},"ts":"1731061269708"} 2024-11-08T10:21:09,710 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:21:09,712 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:21:09,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1610 sec 2024-11-08T10:21:10,638 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:21:11,478 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087911 expired clientIPAndPort=172.17.0.2:57478, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T10:21:11,556 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-08T10:21:11,557 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:21:11,557 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:11,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-08T10:21:11,561 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463760 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:11,562 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T10:21:11,562 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:11,562 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-08T10:21:11,564 DEBUG [hconnection-0x69a20f29-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:11,566 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53962, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:11,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:21:11,575 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:21:11,577 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:21:11,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463763 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:11,578 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:21:11,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463763 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:11,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T10:21:11,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-08T10:21:13,889 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:21:14,581 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4012: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1., hostname=2db67accb017,37913,1731061095589, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4012: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:21:14,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3003 2024-11-08T10:21:14,582 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3dc94bc0 to 127.0.0.1:50547 2024-11-08T10:21:14,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:14,582 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:14,582 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:14,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:14,582 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b578082 to 127.0.0.1:50547 2024-11-08T10:21:14,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 203), ProcessCount=11 (was 11), AvailableMemoryMB=3139 (was 3167) 2024-11-08T10:21:14,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=3138 2024-11-08T10:21:14,601 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x036fcb14 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52027f3a 2024-11-08T10:21:14,604 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x036fcb14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4167654a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:14,605 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22778866 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11749517 2024-11-08T10:21:14,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbecb46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:14,610 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:21:14,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T10:21:14,612 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:21:14,612 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:14,612 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-08T10:21:14,612 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:21:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T10:21:14,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741885_1061 (size=421) 2024-11-08T10:21:15,024 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a4af48f1a9f782520909de9bd94b26d2, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:21:15,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741886_1062 (size=72) 2024-11-08T10:21:15,079 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:21:15,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T10:21:15,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:15,431 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:15,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing a4af48f1a9f782520909de9bd94b26d2, disabling compactions & flushes 2024-11-08T10:21:15,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. after waiting 0 ms 2024-11-08T10:21:15,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for a4af48f1a9f782520909de9bd94b26d2: 2024-11-08T10:21:15,433 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:21:15,433 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731061275433"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061275433"}]},"ts":"1731061275433"} 2024-11-08T10:21:15,435 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:21:15,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:21:15,436 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061275435"}]},"ts":"1731061275435"} 2024-11-08T10:21:15,437 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:21:15,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a4af48f1a9f782520909de9bd94b26d2, ASSIGN}] 2024-11-08T10:21:15,441 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a4af48f1a9f782520909de9bd94b26d2, ASSIGN 2024-11-08T10:21:15,441 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a4af48f1a9f782520909de9bd94b26d2, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:21:15,592 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=a4af48f1a9f782520909de9bd94b26d2, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:15,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure a4af48f1a9f782520909de9bd94b26d2, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:21:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T10:21:15,746 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:21:15,750 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => a4af48f1a9f782520909de9bd94b26d2, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:21:15,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:15,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,753 INFO [StoreOpener-a4af48f1a9f782520909de9bd94b26d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,755 INFO [StoreOpener-a4af48f1a9f782520909de9bd94b26d2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4af48f1a9f782520909de9bd94b26d2 columnFamilyName testFamily 2024-11-08T10:21:15,755 DEBUG [StoreOpener-a4af48f1a9f782520909de9bd94b26d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:15,755 INFO [StoreOpener-a4af48f1a9f782520909de9bd94b26d2-1 {}] regionserver.HStore(327): Store=a4af48f1a9f782520909de9bd94b26d2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:21:15,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:15,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:21:15,762 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened a4af48f1a9f782520909de9bd94b26d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58891945, jitterRate=-0.12244163453578949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:21:15,764 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for a4af48f1a9f782520909de9bd94b26d2: 2024-11-08T10:21:15,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2., pid=83, masterSystemTime=1731061275745 2024-11-08T10:21:15,766 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,766 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:21:15,766 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=a4af48f1a9f782520909de9bd94b26d2, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:15,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T10:21:15,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure a4af48f1a9f782520909de9bd94b26d2, server=2db67accb017,37913,1731061095589 in 174 msec 2024-11-08T10:21:15,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T10:21:15,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a4af48f1a9f782520909de9bd94b26d2, ASSIGN in 329 msec 2024-11-08T10:21:15,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:21:15,771 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061275771"}]},"ts":"1731061275771"} 2024-11-08T10:21:15,772 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:21:15,774 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:21:15,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1650 sec 2024-11-08T10:21:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T10:21:17,622 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-08T10:21:17,622 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:21:17,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:17,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-08T10:21:17,626 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463766 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:17,627 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T10:21:17,627 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:17,627 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T10:21:17,629 DEBUG [hconnection-0x3487eb8a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:17,630 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53966, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:17,637 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:21:17,639 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:21:17,641 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:21:17,641 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T10:21:17,974 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-08T10:21:19,585 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087897 expired clientIPAndPort=172.17.0.2:59740, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:21,296 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:21:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-08T10:21:24,657 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:21:24,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:53966 deadline: 1731061286640, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T10:21:24,658 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T10:21:24,659 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 13538939168532463769 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T10:21:24,659 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7020 2024-11-08T10:21:24,659 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22778866 to 127.0.0.1:50547 2024-11-08T10:21:24,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:24,660 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:24,660 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:24,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:24,660 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x036fcb14 to 127.0.0.1:50547 2024-11-08T10:21:24,669 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=173 (was 187), ProcessCount=11 (was 11), AvailableMemoryMB=3011 (was 3138) 2024-11-08T10:21:24,677 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=173, ProcessCount=11, AvailableMemoryMB=3010 2024-11-08T10:21:24,678 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ae0d2be to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2001144c 2024-11-08T10:21:24,682 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x3ae0d2be {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea9e1f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:24,683 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07cf26a8 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@73c7687c 2024-11-08T10:21:24,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f8e3198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:24,688 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:21:24,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T10:21:24,690 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:21:24,690 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:24,691 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-08T10:21:24,691 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:21:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T10:21:24,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741887_1063 (size=427) 2024-11-08T10:21:25,100 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c39a56b147dd6b099a315e1ebc0190be, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:21:25,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741888_1064 (size=78) 2024-11-08T10:21:25,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T10:21:25,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:25,508 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:25,509 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing c39a56b147dd6b099a315e1ebc0190be, disabling compactions & flushes 2024-11-08T10:21:25,509 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,509 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,509 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. after waiting 0 ms 2024-11-08T10:21:25,509 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,509 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,509 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for c39a56b147dd6b099a315e1ebc0190be: 2024-11-08T10:21:25,510 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:21:25,510 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731061285510"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061285510"}]},"ts":"1731061285510"} 2024-11-08T10:21:25,512 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:21:25,513 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:21:25,513 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061285513"}]},"ts":"1731061285513"} 2024-11-08T10:21:25,514 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:21:25,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c39a56b147dd6b099a315e1ebc0190be, ASSIGN}] 2024-11-08T10:21:25,520 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c39a56b147dd6b099a315e1ebc0190be, ASSIGN 2024-11-08T10:21:25,521 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c39a56b147dd6b099a315e1ebc0190be, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:21:25,671 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=c39a56b147dd6b099a315e1ebc0190be, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:25,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure c39a56b147dd6b099a315e1ebc0190be, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:21:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T10:21:25,826 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:21:25,831 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,831 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => c39a56b147dd6b099a315e1ebc0190be, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:21:25,831 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,831 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:25,832 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,832 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,833 INFO [StoreOpener-c39a56b147dd6b099a315e1ebc0190be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,835 INFO [StoreOpener-c39a56b147dd6b099a315e1ebc0190be-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39a56b147dd6b099a315e1ebc0190be columnFamilyName testFamily 2024-11-08T10:21:25,835 DEBUG [StoreOpener-c39a56b147dd6b099a315e1ebc0190be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:25,836 INFO [StoreOpener-c39a56b147dd6b099a315e1ebc0190be-1 {}] regionserver.HStore(327): Store=c39a56b147dd6b099a315e1ebc0190be/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:21:25,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,839 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:25,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:21:25,841 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened c39a56b147dd6b099a315e1ebc0190be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68715415, jitterRate=0.023939475417137146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:21:25,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for c39a56b147dd6b099a315e1ebc0190be: 2024-11-08T10:21:25,843 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., pid=86, masterSystemTime=1731061285825 2024-11-08T10:21:25,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,845 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:21:25,845 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=c39a56b147dd6b099a315e1ebc0190be, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:25,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T10:21:25,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure c39a56b147dd6b099a315e1ebc0190be, server=2db67accb017,37913,1731061095589 in 173 msec 2024-11-08T10:21:25,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T10:21:25,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c39a56b147dd6b099a315e1ebc0190be, ASSIGN in 330 msec 2024-11-08T10:21:25,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:21:25,851 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061285851"}]},"ts":"1731061285851"} 2024-11-08T10:21:25,853 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:21:25,855 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:21:25,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1660 sec 2024-11-08T10:21:26,591 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087895 expired clientIPAndPort=172.17.0.2:59740, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T10:21:27,695 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-08T10:21:27,695 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:21:27,695 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:27,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-08T10:21:27,699 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463771 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:27,701 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T10:21:27,701 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:27,701 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T10:21:27,703 DEBUG [hconnection-0x27c61c23-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:27,705 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41372, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:27,711 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:21:27,713 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:21:27,714 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:21:27,715 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x3ae0d2be {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:21:27,715 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x3ae0d2be {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:27,716 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41386, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:21:27,718 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:21:29,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T10:21:31,333 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:21:33,721 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2024-11-08T10:21:33,721 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2024-11-08T10:21:33,721 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:21:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T10:21:34,414 ERROR [LeaseRenewer:jenkins@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46031,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:21:34,697 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087890 expired clientIPAndPort=172.17.0.2:49690, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:21:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:21:34,741 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T10:21:35,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T10:21:35,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:36,721 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1972ms 2024-11-08T10:21:36,721 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1972ms 2024-11-08T10:21:36,721 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:21:36,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T10:21:33.721Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6003ms, rpcTimeout=6000ms 2024-11-08T10:21:36.721Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1972ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1972ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1972ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T10:21:36,722 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-08T10:21:36,722 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x07cf26a8 to 127.0.0.1:50547 2024-11-08T10:21:36,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:36,723 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:36,723 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:36,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:36,723 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3ae0d2be to 127.0.0.1:50547 2024-11-08T10:21:36,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=226 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 173), ProcessCount=11 (was 11), AvailableMemoryMB=3041 (was 3010) - AvailableMemoryMB LEAK? - 2024-11-08T10:21:36,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=154, ProcessCount=11, AvailableMemoryMB=3040 2024-11-08T10:21:36,741 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23b61127 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@340dc796 2024-11-08T10:21:36,747 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x23b61127 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bd6a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:36,748 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x04fb5005 to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e15f901 2024-11-08T10:21:36,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db9eecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:36,753 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:21:36,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-08T10:21:36,755 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:21:36,755 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:36,755 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-08T10:21:36,755 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:21:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T10:21:36,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741889_1065 (size=415) 2024-11-08T10:21:37,165 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a45a033254af53ee4f828971c00bd5c3, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:21:37,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741890_1066 (size=66) 2024-11-08T10:21:37,300 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087883 expired clientIPAndPort=172.17.0.2:44088, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:37,300 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087884 expired clientIPAndPort=172.17.0.2:44088, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:37,300 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087886 expired clientIPAndPort=172.17.0.2:44088, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:37,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:37,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing a45a033254af53ee4f828971c00bd5c3, disabling compactions & flushes 2024-11-08T10:21:37,572 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. after waiting 0 ms 2024-11-08T10:21:37,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,572 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,572 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for a45a033254af53ee4f828971c00bd5c3: 2024-11-08T10:21:37,574 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:21:37,574 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731061297574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061297574"}]},"ts":"1731061297574"} 2024-11-08T10:21:37,575 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:21:37,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:21:37,576 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061297576"}]},"ts":"1731061297576"} 2024-11-08T10:21:37,577 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:21:37,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=a45a033254af53ee4f828971c00bd5c3, ASSIGN}] 2024-11-08T10:21:37,582 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=a45a033254af53ee4f828971c00bd5c3, ASSIGN 2024-11-08T10:21:37,583 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=a45a033254af53ee4f828971c00bd5c3, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:21:37,733 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=a45a033254af53ee4f828971c00bd5c3, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:37,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure a45a033254af53ee4f828971c00bd5c3, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:21:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T10:21:37,887 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:21:37,893 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => a45a033254af53ee4f828971c00bd5c3, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:21:37,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:37,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,895 INFO [StoreOpener-a45a033254af53ee4f828971c00bd5c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,896 INFO [StoreOpener-a45a033254af53ee4f828971c00bd5c3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a45a033254af53ee4f828971c00bd5c3 columnFamilyName testFamily 2024-11-08T10:21:37,896 DEBUG [StoreOpener-a45a033254af53ee4f828971c00bd5c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:37,897 INFO [StoreOpener-a45a033254af53ee4f828971c00bd5c3-1 {}] regionserver.HStore(327): Store=a45a033254af53ee4f828971c00bd5c3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:21:37,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,898 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,899 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:37,902 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:21:37,902 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened a45a033254af53ee4f828971c00bd5c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60387367, jitterRate=-0.10015811026096344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:21:37,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for a45a033254af53ee4f828971c00bd5c3: 2024-11-08T10:21:37,903 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3., pid=89, masterSystemTime=1731061297887 2024-11-08T10:21:37,905 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,905 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:21:37,905 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=a45a033254af53ee4f828971c00bd5c3, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:37,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T10:21:37,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure a45a033254af53ee4f828971c00bd5c3, server=2db67accb017,37913,1731061095589 in 172 msec 2024-11-08T10:21:37,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-08T10:21:37,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=a45a033254af53ee4f828971c00bd5c3, ASSIGN in 327 msec 2024-11-08T10:21:37,909 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:21:37,910 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061297910"}]},"ts":"1731061297910"} 2024-11-08T10:21:37,911 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:21:37,913 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:21:37,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1600 sec 2024-11-08T10:21:39,702 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46031 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46031,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:21:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T10:21:39,759 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-08T10:21:39,759 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:21:39,759 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:39,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-08T10:21:39,763 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463778 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:39,765 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T10:21:39,765 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:39,765 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-08T10:21:39,767 DEBUG [hconnection-0x38caa122-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:39,768 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39458, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:39,775 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:21:39,777 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T10:21:39,778 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T10:21:39,778 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463781 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:39,779 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T10:21:39,780 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463781 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:39,780 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T10:21:41,242 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:21:41,977 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-08T10:21:42,304 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087877 expired clientIPAndPort=172.17.0.2:46122, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:21:43,429 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:21:43,889 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T10:21:45,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T10:21:45,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:46,908 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087872 expired clientIPAndPort=172.17.0.2:46124, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:21:48,781 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3., hostname=2db67accb017,37913,1731061095589, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3., hostname=2db67accb017,37913,1731061095589, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T10:21:48,782 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9002 2024-11-08T10:21:48,782 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x04fb5005 to 127.0.0.1:50547 2024-11-08T10:21:48,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:48,782 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:48,782 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:48,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:48,782 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x23b61127 to 127.0.0.1:50547 2024-11-08T10:21:48,791 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=227 (was 226) - Thread LEAK? -, OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=120 (was 154), ProcessCount=11 (was 11), AvailableMemoryMB=3040 (was 3040) 2024-11-08T10:21:48,799 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=120, ProcessCount=11, AvailableMemoryMB=3040 2024-11-08T10:21:48,801 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59d1c29a to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38915703 2024-11-08T10:21:48,804 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x59d1c29a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc65526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:48,805 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c71e86a to 127.0.0.1:50547 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3272664b 2024-11-08T10:21:48,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e0b78d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:21:48,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:21:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T10:21:48,812 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:21:48,812 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:48,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-08T10:21:48,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:21:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T10:21:48,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741891_1067 (size=431) 2024-11-08T10:21:48,909 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087870 expired clientIPAndPort=172.17.0.2:46124, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:21:49,223 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 517b4604f72f9dd7b55701f7411e363a, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01 2024-11-08T10:21:49,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741892_1068 (size=82) 2024-11-08T10:21:49,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:49,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 517b4604f72f9dd7b55701f7411e363a, disabling compactions & flushes 2024-11-08T10:21:49,230 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. after waiting 0 ms 2024-11-08T10:21:49,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,230 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 517b4604f72f9dd7b55701f7411e363a: 2024-11-08T10:21:49,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:21:49,231 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731061309231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061309231"}]},"ts":"1731061309231"} 2024-11-08T10:21:49,233 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T10:21:49,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:21:49,234 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061309233"}]},"ts":"1731061309233"} 2024-11-08T10:21:49,235 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T10:21:49,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=517b4604f72f9dd7b55701f7411e363a, ASSIGN}] 2024-11-08T10:21:49,240 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=517b4604f72f9dd7b55701f7411e363a, ASSIGN 2024-11-08T10:21:49,241 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=517b4604f72f9dd7b55701f7411e363a, ASSIGN; state=OFFLINE, location=2db67accb017,37913,1731061095589; forceNewPlan=false, retain=false 2024-11-08T10:21:49,280 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:21:49,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=517b4604f72f9dd7b55701f7411e363a, regionState=OPENING, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:49,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 517b4604f72f9dd7b55701f7411e363a, server=2db67accb017,37913,1731061095589}] 2024-11-08T10:21:49,545 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 2db67accb017,37913,1731061095589 2024-11-08T10:21:49,550 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,551 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 517b4604f72f9dd7b55701f7411e363a, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:21:49,551 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,551 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:21:49,551 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,551 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,553 INFO [StoreOpener-517b4604f72f9dd7b55701f7411e363a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,554 INFO [StoreOpener-517b4604f72f9dd7b55701f7411e363a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 517b4604f72f9dd7b55701f7411e363a columnFamilyName testFamily 2024-11-08T10:21:49,554 DEBUG [StoreOpener-517b4604f72f9dd7b55701f7411e363a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:21:49,555 INFO [StoreOpener-517b4604f72f9dd7b55701f7411e363a-1 {}] regionserver.HStore(327): Store=517b4604f72f9dd7b55701f7411e363a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:21:49,556 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,556 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,558 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:49,560 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:21:49,561 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 517b4604f72f9dd7b55701f7411e363a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69636944, jitterRate=0.03767132759094238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:21:49,561 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 517b4604f72f9dd7b55701f7411e363a: 2024-11-08T10:21:49,562 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., pid=92, masterSystemTime=1731061309545 2024-11-08T10:21:49,564 DEBUG [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,564 INFO [RS_OPEN_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:21:49,564 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=517b4604f72f9dd7b55701f7411e363a, regionState=OPEN, openSeqNum=2, regionLocation=2db67accb017,37913,1731061095589 2024-11-08T10:21:49,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-08T10:21:49,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 517b4604f72f9dd7b55701f7411e363a, server=2db67accb017,37913,1731061095589 in 172 msec 2024-11-08T10:21:49,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T10:21:49,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=517b4604f72f9dd7b55701f7411e363a, ASSIGN in 327 msec 2024-11-08T10:21:49,568 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:21:49,568 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061309568"}]},"ts":"1731061309568"} 2024-11-08T10:21:49,569 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T10:21:49,572 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:21:49,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 762 msec 2024-11-08T10:21:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40511 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T10:21:49,814 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-08T10:21:49,815 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T10:21:49,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:49,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37913 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-08T10:21:49,820 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 13538939168532463782 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T10:21:49,821 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T10:21:49,821 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:21:49,821 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T10:21:49,823 DEBUG [hconnection-0x1c0e03a1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:49,824 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37596, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:49,831 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T10:21:49,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T10:21:49,833 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T10:21:49,834 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x59d1c29a {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2db67accb017,37913,1731061095589, seqNum=-1] 2024-11-08T10:21:49,834 DEBUG [ReadOnlyZKClient-127.0.0.1:50547@0x59d1c29a {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:21:49,835 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37604, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:21:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:21:49,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:21:50,841 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-08T10:21:50,841 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-08T10:21:50,841 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:21:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-08T10:21:51,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2] 2024-11-08T10:21:51,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T10:21:52,840 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=972ms 2024-11-08T10:21:52,840 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=972ms 2024-11-08T10:21:52,840 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., hostname=2db67accb017,37913,1731061095589, seqNum=2 from cache 2024-11-08T10:21:52,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T10:21:50.841Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-08T10:21:52.840Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=972ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2db67accb017:37913 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=972ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=972ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T10:21:52,842 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3008 2024-11-08T10:21:52,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c71e86a to 127.0.0.1:50547 2024-11-08T10:21:52,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:52,842 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T10:21:52,842 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:21:52,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:52,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59d1c29a to 127.0.0.1:50547 2024-11-08T10:21:52,851 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=233 (was 227) - Thread LEAK? -, OpenFileDescriptor=450 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=120 (was 120), ProcessCount=11 (was 11), AvailableMemoryMB=3035 (was 3040) 2024-11-08T10:21:52,851 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-08T10:21:52,851 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T10:21:52,852 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60b11d5f to 127.0.0.1:50547 2024-11-08T10:21:52,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:52,852 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T10:21:52,852 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=468212891, stopped=false 2024-11-08T10:21:52,852 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=2db67accb017,40511,1731061094572 2024-11-08T10:21:52,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:21:52,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:21:52,854 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-08T10:21:52,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:21:52,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:21:52,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:52,854 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '2db67accb017,37913,1731061095589' ***** 2024-11-08T10:21:52,854 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-08T10:21:52,855 INFO [RS:0;2db67accb017:37913 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T10:21:52,855 INFO [RS:0;2db67accb017:37913 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T10:21:52,855 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-08T10:21:52,855 INFO [RS:0;2db67accb017:37913 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T10:21:52,855 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:21:52,855 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for b0d2d9924440ee776e54c19a9c601b36 2024-11-08T10:21:52,855 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 4e72ba4990ef0072afcae1a4ad9ca190 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 5c9bae98c605ef30875f4e0115c93556 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for ec472e3a7c3c4010c8c2b8708ba386f2 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 9a51f8d80633f3cc5b5a69d57da95629 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 18632d949262e24bebe96198d33fa2ce 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 989074082986a01232c976f6de58a84d 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 4161700af8f79081e6d959b624f78cc1 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 182d6d30146365e85f2f0b8f61460763 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for dc59aab9dd788edfed6060a8dd176fad 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 7574e219918a9b10e7232c85198e870b 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 75db9bee13abc31f9df9651eb584b046 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for ce9524b9626bb8e191e7ad056f5d2da8 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 54d468ed537025de01d7e24f13cf3a2e 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 7a9a8e7ae601fb76ff6bcacfb7f43b8c 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 1ba716bec7899a38c7fd594140143301 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 8ba0428adc53ccbb2911449553de4eb7 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 00032c2457ddc4b4cee4c234010c2624 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 43a73611ad06dc7aebd9e66c4bb58558 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 68acff9b3e69cb0077e2babe1fe22e73 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 517b4604f72f9dd7b55701f7411e363a 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for c39a56b147dd6b099a315e1ebc0190be 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for a45a033254af53ee4f828971c00bd5c3 2024-11-08T10:21:52,856 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b0d2d9924440ee776e54c19a9c601b36, disabling compactions & flushes 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for d1ef8d97072ec94ee606727c3fa0252e 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:52,856 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for c98f706b3d1970465cfb9fe302b92eeb 2024-11-08T10:21:52,856 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 645e70ee8fcf32e5b76639c429a17b6d 2024-11-08T10:21:52,856 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. after waiting 0 ms 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:21:52,856 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for a4af48f1a9f782520909de9bd94b26d2 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1224): stopping server 2db67accb017,37913,1731061095589 2024-11-08T10:21:52,856 DEBUG [RS:0;2db67accb017:37913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T10:21:52,856 INFO [RS:0;2db67accb017:37913 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T10:21:52,857 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-08T10:21:52,857 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b0d2d9924440ee776e54c19a9c601b36 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:52,857 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-08T10:21:52,857 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:21:52,857 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T10:21:52,857 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T10:21:52,857 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:21:52,857 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:21:52,857 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-08T10:21:52,857 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1588230740, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 4e72ba4990ef0072afcae1a4ad9ca190, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, b0d2d9924440ee776e54c19a9c601b36, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:52,915 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/.tmp/testFamily/8a1d71b01e0643a1ab37641fd395eeea is 61, key is row-0/testFamily:testQualifier/1731061180550/Put/seqid=0 2024-11-08T10:21:52,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741893_1069 (size=5049) 2024-11-08T10:21:52,930 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/.tmp/info/e1e18d2adcc54751a85b1b65fdeaa6db is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad./info:regioninfo/1731061129921/Put/seqid=0 2024-11-08T10:21:52,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741894_1070 (size=51873) 2024-11-08T10:21:53,058 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1588230740, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 4e72ba4990ef0072afcae1a4ad9ca190, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, b0d2d9924440ee776e54c19a9c601b36, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:53,105 INFO [regionserver/2db67accb017:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T10:21:53,105 INFO [regionserver/2db67accb017:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T10:21:53,258 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1588230740, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 4e72ba4990ef0072afcae1a4ad9ca190, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, b0d2d9924440ee776e54c19a9c601b36, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:53,328 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/.tmp/testFamily/8a1d71b01e0643a1ab37641fd395eeea 2024-11-08T10:21:53,335 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/.tmp/info/e1e18d2adcc54751a85b1b65fdeaa6db 2024-11-08T10:21:53,362 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T10:21:53,374 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/.tmp/testFamily/8a1d71b01e0643a1ab37641fd395eeea as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/testFamily/8a1d71b01e0643a1ab37641fd395eeea 2024-11-08T10:21:53,382 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/testFamily/8a1d71b01e0643a1ab37641fd395eeea, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:53,384 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b0d2d9924440ee776e54c19a9c601b36 in 527ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:53,392 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/.tmp/table/41275c78e4b2435db4f7e2f17ca42455 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731061129939/Put/seqid=0 2024-11-08T10:21:53,393 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/b0d2d9924440ee776e54c19a9c601b36/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:53,395 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:21:53,396 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b0d2d9924440ee776e54c19a9c601b36: 2024-11-08T10:21:53,396 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731061177527.b0d2d9924440ee776e54c19a9c601b36. 2024-11-08T10:21:53,396 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4e72ba4990ef0072afcae1a4ad9ca190, disabling compactions & flushes 2024-11-08T10:21:53,396 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:21:53,396 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:21:53,396 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. after waiting 0 ms 2024-11-08T10:21:53,396 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:21:53,396 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4e72ba4990ef0072afcae1a4ad9ca190 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:53,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741895_1071 (size=9821) 2024-11-08T10:21:53,416 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/.tmp/testFamily/ab611a431d8e4e978f790e50f0c049af is 61, key is row-0/testFamily:testQualifier/1731061225118/Put/seqid=0 2024-11-08T10:21:53,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741896_1072 (size=5049) 2024-11-08T10:21:53,458 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1588230740, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 4e72ba4990ef0072afcae1a4ad9ca190, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:53,659 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1588230740, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 4e72ba4990ef0072afcae1a4ad9ca190, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:53,803 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/.tmp/table/41275c78e4b2435db4f7e2f17ca42455 2024-11-08T10:21:53,812 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/.tmp/info/e1e18d2adcc54751a85b1b65fdeaa6db as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/info/e1e18d2adcc54751a85b1b65fdeaa6db 2024-11-08T10:21:53,820 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/info/e1e18d2adcc54751a85b1b65fdeaa6db, entries=290, sequenceid=149, filesize=50.7 K 2024-11-08T10:21:53,821 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/.tmp/testFamily/ab611a431d8e4e978f790e50f0c049af 2024-11-08T10:21:53,822 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/.tmp/table/41275c78e4b2435db4f7e2f17ca42455 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/table/41275c78e4b2435db4f7e2f17ca42455 2024-11-08T10:21:53,829 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/.tmp/testFamily/ab611a431d8e4e978f790e50f0c049af as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/testFamily/ab611a431d8e4e978f790e50f0c049af 2024-11-08T10:21:53,831 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/table/41275c78e4b2435db4f7e2f17ca42455, entries=58, sequenceid=149, filesize=9.6 K 2024-11-08T10:21:53,832 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 975ms, sequenceid=149, compaction requested=false 2024-11-08T10:21:53,837 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/testFamily/ab611a431d8e4e978f790e50f0c049af, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:53,838 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4e72ba4990ef0072afcae1a4ad9ca190 in 441ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:53,845 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-08T10:21:53,845 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:21:53,845 INFO [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T10:21:53,845 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T10:21:53,846 DEBUG [RS_CLOSE_META-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T10:21:53,846 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/4e72ba4990ef0072afcae1a4ad9ca190/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:53,847 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:21:53,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4e72ba4990ef0072afcae1a4ad9ca190: 2024-11-08T10:21:53,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731061222102.4e72ba4990ef0072afcae1a4ad9ca190. 2024-11-08T10:21:53,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5c9bae98c605ef30875f4e0115c93556, disabling compactions & flushes 2024-11-08T10:21:53,848 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:53,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:53,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. after waiting 0 ms 2024-11-08T10:21:53,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:53,848 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 5c9bae98c605ef30875f4e0115c93556 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:53,859 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-08T10:21:53,859 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:53,867 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/.tmp/testFamily/6d90b8678b1c4da39d0a8e535d2a8f85 is 61, key is row-0/testFamily:testQualifier/1731061237190/Put/seqid=0 2024-11-08T10:21:53,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741897_1073 (size=5049) 2024-11-08T10:21:53,979 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T10:21:54,059 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:54,260 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 5c9bae98c605ef30875f4e0115c93556, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:54,273 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/.tmp/testFamily/6d90b8678b1c4da39d0a8e535d2a8f85 2024-11-08T10:21:54,281 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/.tmp/testFamily/6d90b8678b1c4da39d0a8e535d2a8f85 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/testFamily/6d90b8678b1c4da39d0a8e535d2a8f85 2024-11-08T10:21:54,288 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/testFamily/6d90b8678b1c4da39d0a8e535d2a8f85, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:54,289 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5c9bae98c605ef30875f4e0115c93556 in 441ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:54,293 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_1_-true/5c9bae98c605ef30875f4e0115c93556/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:54,294 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:54,294 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5c9bae98c605ef30875f4e0115c93556: 2024-11-08T10:21:54,294 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731061234173.5c9bae98c605ef30875f4e0115c93556. 2024-11-08T10:21:54,294 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ec472e3a7c3c4010c8c2b8708ba386f2, disabling compactions & flushes 2024-11-08T10:21:54,294 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:21:54,294 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:21:54,295 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. after waiting 1 ms 2024-11-08T10:21:54,295 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:21:54,295 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ec472e3a7c3c4010c8c2b8708ba386f2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:54,317 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/.tmp/testFamily/102247f275f04a69bcb06bda882399e0 is 61, key is row-0/testFamily:testQualifier/1731061251394/Put/seqid=0 2024-11-08T10:21:54,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741898_1074 (size=5049) 2024-11-08T10:21:54,460 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:54,661 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, ec472e3a7c3c4010c8c2b8708ba386f2, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:54,723 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/.tmp/testFamily/102247f275f04a69bcb06bda882399e0 2024-11-08T10:21:54,732 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/.tmp/testFamily/102247f275f04a69bcb06bda882399e0 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/testFamily/102247f275f04a69bcb06bda882399e0 2024-11-08T10:21:54,739 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/testFamily/102247f275f04a69bcb06bda882399e0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:54,742 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ec472e3a7c3c4010c8c2b8708ba386f2 in 447ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:54,747 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_1_-true/ec472e3a7c3c4010c8c2b8708ba386f2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:54,748 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:21:54,748 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ec472e3a7c3c4010c8c2b8708ba386f2: 2024-11-08T10:21:54,748 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:21:54,748 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9a51f8d80633f3cc5b5a69d57da95629, disabling compactions & flushes 2024-11-08T10:21:54,748 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:54,748 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:54,748 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. after waiting 0 ms 2024-11-08T10:21:54,748 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:54,748 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 9a51f8d80633f3cc5b5a69d57da95629 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:54,770 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/.tmp/testFamily/f754e353f87944fbb2fb0241ae80b532 is 61, key is row-0/testFamily:testQualifier/1731061207898/Put/seqid=0 2024-11-08T10:21:54,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741899_1075 (size=5049) 2024-11-08T10:21:54,861 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-08T10:21:54,861 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:54,914 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -4907804905177087865 expired clientIPAndPort=172.17.0.2:52916, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. 2024-11-08T10:21:54,914 ERROR [regionserver/2db67accb017:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(581): Closing scanner -4907804905177087865 clientIPAndPort=172.17.0.2:52916, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. org.apache.hadoop.hbase.NotServingRegionException: testNormalScanTimeoutOnNextAsync_1_-true,,1731061248379.ec472e3a7c3c4010c8c2b8708ba386f2. is not online on 2db67accb017,37913,1731061095589 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3673) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3651) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices$ScannerListener.leaseExpired(RSRpcServices.java:576) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:107) ~[classes/:?] 2024-11-08T10:21:55,061 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, 9a51f8d80633f3cc5b5a69d57da95629, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:55,085 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:21:55,175 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/.tmp/testFamily/f754e353f87944fbb2fb0241ae80b532 2024-11-08T10:21:55,183 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/.tmp/testFamily/f754e353f87944fbb2fb0241ae80b532 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/testFamily/f754e353f87944fbb2fb0241ae80b532 2024-11-08T10:21:55,189 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/testFamily/f754e353f87944fbb2fb0241ae80b532, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:55,190 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9a51f8d80633f3cc5b5a69d57da95629 in 442ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:55,194 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/9a51f8d80633f3cc5b5a69d57da95629/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:55,195 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:55,195 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9a51f8d80633f3cc5b5a69d57da95629: 2024-11-08T10:21:55,195 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731061206879.9a51f8d80633f3cc5b5a69d57da95629. 2024-11-08T10:21:55,195 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 18632d949262e24bebe96198d33fa2ce, disabling compactions & flushes 2024-11-08T10:21:55,195 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:21:55,196 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:21:55,196 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. after waiting 0 ms 2024-11-08T10:21:55,196 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:21:55,196 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 18632d949262e24bebe96198d33fa2ce 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:55,211 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/.tmp/testFamily/9b63053c850c45d5894dd3973ac42537 is 61, key is row-0/testFamily:testQualifier/1731061257459/Put/seqid=0 2024-11-08T10:21:55,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741900_1076 (size=5049) 2024-11-08T10:21:55,228 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T10:21:55,228 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:21:55,229 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T10:21:55,261 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:55,462 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 18632d949262e24bebe96198d33fa2ce, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:55,617 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/.tmp/testFamily/9b63053c850c45d5894dd3973ac42537 2024-11-08T10:21:55,625 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/.tmp/testFamily/9b63053c850c45d5894dd3973ac42537 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/testFamily/9b63053c850c45d5894dd3973ac42537 2024-11-08T10:21:55,632 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/testFamily/9b63053c850c45d5894dd3973ac42537, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:55,633 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 18632d949262e24bebe96198d33fa2ce in 437ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:55,638 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_1_-true/18632d949262e24bebe96198d33fa2ce/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:55,639 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:21:55,639 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 18632d949262e24bebe96198d33fa2ce: 2024-11-08T10:21:55,639 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731061254439.18632d949262e24bebe96198d33fa2ce. 2024-11-08T10:21:55,639 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 989074082986a01232c976f6de58a84d, disabling compactions & flushes 2024-11-08T10:21:55,639 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:21:55,639 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:21:55,639 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. after waiting 0 ms 2024-11-08T10:21:55,639 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:21:55,639 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 989074082986a01232c976f6de58a84d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:55,658 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/.tmp/testFamily/cf3177b2f5474eb7a23a22bae98ad6be is 61, key is row-0/testFamily:testQualifier/1731061173856/Put/seqid=0 2024-11-08T10:21:55,662 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:55,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741901_1077 (size=5049) 2024-11-08T10:21:55,862 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-08T10:21:55,862 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:56,063 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, 989074082986a01232c976f6de58a84d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:56,063 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/.tmp/testFamily/cf3177b2f5474eb7a23a22bae98ad6be 2024-11-08T10:21:56,071 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/.tmp/testFamily/cf3177b2f5474eb7a23a22bae98ad6be as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/testFamily/cf3177b2f5474eb7a23a22bae98ad6be 2024-11-08T10:21:56,078 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/testFamily/cf3177b2f5474eb7a23a22bae98ad6be, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:56,079 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 989074082986a01232c976f6de58a84d in 440ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:56,083 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_0_-false/989074082986a01232c976f6de58a84d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:56,084 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:21:56,084 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 989074082986a01232c976f6de58a84d: 2024-11-08T10:21:56,084 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731061172833.989074082986a01232c976f6de58a84d. 2024-11-08T10:21:56,084 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4161700af8f79081e6d959b624f78cc1, disabling compactions & flushes 2024-11-08T10:21:56,084 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:56,084 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:56,084 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. after waiting 0 ms 2024-11-08T10:21:56,084 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:56,084 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4161700af8f79081e6d959b624f78cc1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:56,107 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/.tmp/testFamily/d0f1ee73d1c1402c953a5b927c3d9b1d is 61, key is row-0/testFamily:testQualifier/1731061271567/Put/seqid=0 2024-11-08T10:21:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741902_1078 (size=5049) 2024-11-08T10:21:56,263 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:56,463 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 4161700af8f79081e6d959b624f78cc1, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:56,513 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/.tmp/testFamily/d0f1ee73d1c1402c953a5b927c3d9b1d 2024-11-08T10:21:56,520 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/.tmp/testFamily/d0f1ee73d1c1402c953a5b927c3d9b1d as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/testFamily/d0f1ee73d1c1402c953a5b927c3d9b1d 2024-11-08T10:21:56,526 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/testFamily/d0f1ee73d1c1402c953a5b927c3d9b1d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:56,527 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4161700af8f79081e6d959b624f78cc1 in 443ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:56,531 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_1_-true/4161700af8f79081e6d959b624f78cc1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:56,532 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:56,532 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4161700af8f79081e6d959b624f78cc1: 2024-11-08T10:21:56,532 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731061268550.4161700af8f79081e6d959b624f78cc1. 2024-11-08T10:21:56,532 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 182d6d30146365e85f2f0b8f61460763, disabling compactions & flushes 2024-11-08T10:21:56,532 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:21:56,532 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:21:56,532 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. after waiting 0 ms 2024-11-08T10:21:56,532 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:21:56,532 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 182d6d30146365e85f2f0b8f61460763 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:56,548 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/.tmp/testFamily/124f262864d14b8ea36774ca8377f009 is 61, key is row-0/testFamily:testQualifier/1731061176446/Put/seqid=0 2024-11-08T10:21:56,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741903_1079 (size=5049) 2024-11-08T10:21:56,664 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:56,864 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-08T10:21:56,864 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 182d6d30146365e85f2f0b8f61460763, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:56,954 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/.tmp/testFamily/124f262864d14b8ea36774ca8377f009 2024-11-08T10:21:56,961 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/.tmp/testFamily/124f262864d14b8ea36774ca8377f009 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/testFamily/124f262864d14b8ea36774ca8377f009 2024-11-08T10:21:56,968 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/testFamily/124f262864d14b8ea36774ca8377f009, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:56,969 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 182d6d30146365e85f2f0b8f61460763 in 437ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:56,974 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNext_0_-false/182d6d30146365e85f2f0b8f61460763/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:56,974 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:21:56,974 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 182d6d30146365e85f2f0b8f61460763: 2024-11-08T10:21:56,974 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731061175419.182d6d30146365e85f2f0b8f61460763. 2024-11-08T10:21:56,974 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dc59aab9dd788edfed6060a8dd176fad, disabling compactions & flushes 2024-11-08T10:21:56,974 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:21:56,974 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:21:56,974 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. after waiting 0 ms 2024-11-08T10:21:56,975 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:21:56,975 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing dc59aab9dd788edfed6060a8dd176fad 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:56,991 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/.tmp/testFamily/ac3144c4fd4a49edba8b7c4e459d6830 is 61, key is row-0/testFamily:testQualifier/1731061130504/Put/seqid=0 2024-11-08T10:21:56,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741904_1080 (size=5049) 2024-11-08T10:21:57,064 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:57,265 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, dc59aab9dd788edfed6060a8dd176fad, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:57,397 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/.tmp/testFamily/ac3144c4fd4a49edba8b7c4e459d6830 2024-11-08T10:21:57,404 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/.tmp/testFamily/ac3144c4fd4a49edba8b7c4e459d6830 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/testFamily/ac3144c4fd4a49edba8b7c4e459d6830 2024-11-08T10:21:57,411 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/testFamily/ac3144c4fd4a49edba8b7c4e459d6830, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:57,412 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dc59aab9dd788edfed6060a8dd176fad in 437ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:57,417 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dc59aab9dd788edfed6060a8dd176fad/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:57,417 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:21:57,417 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dc59aab9dd788edfed6060a8dd176fad: 2024-11-08T10:21:57,418 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731061129373.dc59aab9dd788edfed6060a8dd176fad. 2024-11-08T10:21:57,418 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7574e219918a9b10e7232c85198e870b, disabling compactions & flushes 2024-11-08T10:21:57,418 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:21:57,418 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:21:57,418 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. after waiting 0 ms 2024-11-08T10:21:57,418 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:21:57,418 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7574e219918a9b10e7232c85198e870b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:57,436 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/.tmp/testFamily/3a1970a9fdad4a3186d970c8f9e6eeb9 is 61, key is row-0/testFamily:testQualifier/1731061166283/Put/seqid=0 2024-11-08T10:21:57,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741905_1081 (size=5049) 2024-11-08T10:21:57,465 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:57,665 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7574e219918a9b10e7232c85198e870b, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:57,842 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/.tmp/testFamily/3a1970a9fdad4a3186d970c8f9e6eeb9 2024-11-08T10:21:57,848 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/.tmp/testFamily/3a1970a9fdad4a3186d970c8f9e6eeb9 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/testFamily/3a1970a9fdad4a3186d970c8f9e6eeb9 2024-11-08T10:21:57,854 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/testFamily/3a1970a9fdad4a3186d970c8f9e6eeb9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:57,855 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7574e219918a9b10e7232c85198e870b in 437ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:57,859 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnClose_0_-false/7574e219918a9b10e7232c85198e870b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:57,860 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:21:57,860 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7574e219918a9b10e7232c85198e870b: 2024-11-08T10:21:57,860 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731061163263.7574e219918a9b10e7232c85198e870b. 2024-11-08T10:21:57,860 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 75db9bee13abc31f9df9651eb584b046, disabling compactions & flushes 2024-11-08T10:21:57,860 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:57,860 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:57,860 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. after waiting 0 ms 2024-11-08T10:21:57,860 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:57,860 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 75db9bee13abc31f9df9651eb584b046 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:57,865 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 18 regions to close 2024-11-08T10:21:57,865 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:57,876 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/.tmp/testFamily/17c4a9c2a31846eeabb7cf48fe1d489a is 61, key is row-0/testFamily:testQualifier/1731061213059/Put/seqid=0 2024-11-08T10:21:57,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741906_1082 (size=5049) 2024-11-08T10:21:58,066 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:58,266 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 75db9bee13abc31f9df9651eb584b046, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:58,281 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/.tmp/testFamily/17c4a9c2a31846eeabb7cf48fe1d489a 2024-11-08T10:21:58,288 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/.tmp/testFamily/17c4a9c2a31846eeabb7cf48fe1d489a as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/testFamily/17c4a9c2a31846eeabb7cf48fe1d489a 2024-11-08T10:21:58,294 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/testFamily/17c4a9c2a31846eeabb7cf48fe1d489a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:58,295 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 75db9bee13abc31f9df9651eb584b046 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:58,299 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/75db9bee13abc31f9df9651eb584b046/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:58,299 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:58,300 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 75db9bee13abc31f9df9651eb584b046: 2024-11-08T10:21:58,300 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731061212041.75db9bee13abc31f9df9651eb584b046. 2024-11-08T10:21:58,300 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ce9524b9626bb8e191e7ad056f5d2da8, disabling compactions & flushes 2024-11-08T10:21:58,300 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:21:58,300 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:21:58,300 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. after waiting 0 ms 2024-11-08T10:21:58,300 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:21:58,300 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ce9524b9626bb8e191e7ad056f5d2da8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:58,315 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/.tmp/testFamily/829e11f5f1bc4a25b8ae996e1f2e3889 is 61, key is row-0/testFamily:testQualifier/1731061156100/Put/seqid=0 2024-11-08T10:21:58,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741907_1083 (size=5049) 2024-11-08T10:21:58,466 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:58,666 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, ce9524b9626bb8e191e7ad056f5d2da8, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:58,720 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/.tmp/testFamily/829e11f5f1bc4a25b8ae996e1f2e3889 2024-11-08T10:21:58,728 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/.tmp/testFamily/829e11f5f1bc4a25b8ae996e1f2e3889 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/testFamily/829e11f5f1bc4a25b8ae996e1f2e3889 2024-11-08T10:21:58,734 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/testFamily/829e11f5f1bc4a25b8ae996e1f2e3889, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:58,735 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ce9524b9626bb8e191e7ad056f5d2da8 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:58,739 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce9524b9626bb8e191e7ad056f5d2da8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:58,739 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:21:58,739 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ce9524b9626bb8e191e7ad056f5d2da8: 2024-11-08T10:21:58,740 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731061155075.ce9524b9626bb8e191e7ad056f5d2da8. 2024-11-08T10:21:58,740 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 54d468ed537025de01d7e24f13cf3a2e, disabling compactions & flushes 2024-11-08T10:21:58,740 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:21:58,740 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:21:58,740 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. after waiting 0 ms 2024-11-08T10:21:58,740 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:21:58,740 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 54d468ed537025de01d7e24f13cf3a2e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:58,756 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/.tmp/testFamily/c1b6b94ea4194e308ebf02121818b6dd is 61, key is row-0/testFamily:testQualifier/1731061131716/Put/seqid=0 2024-11-08T10:21:58,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741908_1084 (size=5049) 2024-11-08T10:21:58,867 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-08T10:21:58,867 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:59,067 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 54d468ed537025de01d7e24f13cf3a2e, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:59,161 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/.tmp/testFamily/c1b6b94ea4194e308ebf02121818b6dd 2024-11-08T10:21:59,169 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/.tmp/testFamily/c1b6b94ea4194e308ebf02121818b6dd as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/testFamily/c1b6b94ea4194e308ebf02121818b6dd 2024-11-08T10:21:59,175 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/testFamily/c1b6b94ea4194e308ebf02121818b6dd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:59,176 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 54d468ed537025de01d7e24f13cf3a2e in 436ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:59,180 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/54d468ed537025de01d7e24f13cf3a2e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:59,181 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:21:59,181 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 54d468ed537025de01d7e24f13cf3a2e: 2024-11-08T10:21:59,181 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731061130687.54d468ed537025de01d7e24f13cf3a2e. 2024-11-08T10:21:59,181 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7a9a8e7ae601fb76ff6bcacfb7f43b8c, disabling compactions & flushes 2024-11-08T10:21:59,181 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:59,181 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:59,181 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. after waiting 0 ms 2024-11-08T10:21:59,181 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:59,181 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7a9a8e7ae601fb76ff6bcacfb7f43b8c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:59,197 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/.tmp/testFamily/c2daeca85aab42d19adb83d0ebf89562 is 61, key is row-0/testFamily:testQualifier/1731061267013/Put/seqid=0 2024-11-08T10:21:59,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741909_1085 (size=5049) 2024-11-08T10:21:59,267 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:59,468 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 7a9a8e7ae601fb76ff6bcacfb7f43b8c, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:59,602 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/.tmp/testFamily/c2daeca85aab42d19adb83d0ebf89562 2024-11-08T10:21:59,609 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/.tmp/testFamily/c2daeca85aab42d19adb83d0ebf89562 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/testFamily/c2daeca85aab42d19adb83d0ebf89562 2024-11-08T10:21:59,614 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/testFamily/c2daeca85aab42d19adb83d0ebf89562, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:21:59,615 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7a9a8e7ae601fb76ff6bcacfb7f43b8c in 434ms, sequenceid=8, compaction requested=false 2024-11-08T10:21:59,619 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnClose_1_-true/7a9a8e7ae601fb76ff6bcacfb7f43b8c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:21:59,620 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:59,620 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7a9a8e7ae601fb76ff6bcacfb7f43b8c: 2024-11-08T10:21:59,620 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731061263996.7a9a8e7ae601fb76ff6bcacfb7f43b8c. 2024-11-08T10:21:59,620 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1ba716bec7899a38c7fd594140143301, disabling compactions & flushes 2024-11-08T10:21:59,620 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:21:59,620 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:21:59,620 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. after waiting 0 ms 2024-11-08T10:21:59,620 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:21:59,620 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1ba716bec7899a38c7fd594140143301 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:21:59,636 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/.tmp/testFamily/91dc05bde50646239d8594ebefc9c941 is 61, key is row-0/testFamily:testQualifier/1731061190643/Put/seqid=0 2024-11-08T10:21:59,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741910_1086 (size=5049) 2024-11-08T10:21:59,668 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:21:59,868 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 14 regions to close 2024-11-08T10:21:59,868 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 1ba716bec7899a38c7fd594140143301, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:00,041 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/.tmp/testFamily/91dc05bde50646239d8594ebefc9c941 2024-11-08T10:22:00,048 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/.tmp/testFamily/91dc05bde50646239d8594ebefc9c941 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/testFamily/91dc05bde50646239d8594ebefc9c941 2024-11-08T10:22:00,054 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/testFamily/91dc05bde50646239d8594ebefc9c941, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:00,055 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1ba716bec7899a38c7fd594140143301 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:00,059 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/1ba716bec7899a38c7fd594140143301/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:00,060 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:22:00,060 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1ba716bec7899a38c7fd594140143301: 2024-11-08T10:22:00,060 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731061187622.1ba716bec7899a38c7fd594140143301. 2024-11-08T10:22:00,060 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8ba0428adc53ccbb2911449553de4eb7, disabling compactions & flushes 2024-11-08T10:22:00,060 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:22:00,060 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:22:00,060 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. after waiting 0 ms 2024-11-08T10:22:00,060 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:22:00,060 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8ba0428adc53ccbb2911449553de4eb7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:00,068 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:00,076 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/.tmp/testFamily/69d5d84755e44e06b7837f509dda51fe is 61, key is row-0/testFamily:testQualifier/1731061211975/Put/seqid=0 2024-11-08T10:22:00,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741911_1087 (size=5049) 2024-11-08T10:22:00,249 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:22:00,269 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:00,469 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, 8ba0428adc53ccbb2911449553de4eb7, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:00,481 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/.tmp/testFamily/69d5d84755e44e06b7837f509dda51fe 2024-11-08T10:22:00,488 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/.tmp/testFamily/69d5d84755e44e06b7837f509dda51fe as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/testFamily/69d5d84755e44e06b7837f509dda51fe 2024-11-08T10:22:00,494 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/testFamily/69d5d84755e44e06b7837f509dda51fe, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:00,495 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ba0428adc53ccbb2911449553de4eb7 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:00,499 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/8ba0428adc53ccbb2911449553de4eb7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:00,500 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:22:00,500 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8ba0428adc53ccbb2911449553de4eb7: 2024-11-08T10:22:00,500 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731061210953.8ba0428adc53ccbb2911449553de4eb7. 2024-11-08T10:22:00,500 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 00032c2457ddc4b4cee4c234010c2624, disabling compactions & flushes 2024-11-08T10:22:00,500 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:22:00,500 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:22:00,500 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. after waiting 0 ms 2024-11-08T10:22:00,500 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:22:00,500 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 00032c2457ddc4b4cee4c234010c2624 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:00,516 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/.tmp/testFamily/a4a8c9d091b143b986038caeae89e6de is 61, key is row-0/testFamily:testQualifier/1731061200745/Put/seqid=0 2024-11-08T10:22:00,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741912_1088 (size=5049) 2024-11-08T10:22:00,669 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:00,870 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 12 regions to close 2024-11-08T10:22:00,870 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 00032c2457ddc4b4cee4c234010c2624, 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:00,922 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/.tmp/testFamily/a4a8c9d091b143b986038caeae89e6de 2024-11-08T10:22:00,929 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/.tmp/testFamily/a4a8c9d091b143b986038caeae89e6de as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/testFamily/a4a8c9d091b143b986038caeae89e6de 2024-11-08T10:22:00,935 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/testFamily/a4a8c9d091b143b986038caeae89e6de, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:00,936 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 00032c2457ddc4b4cee4c234010c2624 in 436ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:00,940 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_0_-false/00032c2457ddc4b4cee4c234010c2624/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:00,940 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:22:00,940 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 00032c2457ddc4b4cee4c234010c2624: 2024-11-08T10:22:00,940 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731061199723.00032c2457ddc4b4cee4c234010c2624. 2024-11-08T10:22:00,941 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 43a73611ad06dc7aebd9e66c4bb58558, disabling compactions & flushes 2024-11-08T10:22:00,941 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:22:00,941 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:22:00,941 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. after waiting 0 ms 2024-11-08T10:22:00,941 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:22:00,941 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 43a73611ad06dc7aebd9e66c4bb58558 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:00,956 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/.tmp/testFamily/c13b4fd333ef4a0b99b939cf9af46079 is 61, key is row-0/testFamily:testQualifier/1731061245337/Put/seqid=0 2024-11-08T10:22:00,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741913_1089 (size=5049) 2024-11-08T10:22:01,070 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:01,270 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 43a73611ad06dc7aebd9e66c4bb58558, 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:01,361 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/.tmp/testFamily/c13b4fd333ef4a0b99b939cf9af46079 2024-11-08T10:22:01,368 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/.tmp/testFamily/c13b4fd333ef4a0b99b939cf9af46079 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/testFamily/c13b4fd333ef4a0b99b939cf9af46079 2024-11-08T10:22:01,373 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/testFamily/c13b4fd333ef4a0b99b939cf9af46079, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:01,374 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 43a73611ad06dc7aebd9e66c4bb58558 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:01,378 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/43a73611ad06dc7aebd9e66c4bb58558/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:01,379 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:22:01,379 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 43a73611ad06dc7aebd9e66c4bb58558: 2024-11-08T10:22:01,379 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731061242322.43a73611ad06dc7aebd9e66c4bb58558. 2024-11-08T10:22:01,379 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 68acff9b3e69cb0077e2babe1fe22e73, disabling compactions & flushes 2024-11-08T10:22:01,379 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:22:01,379 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:22:01,379 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. after waiting 0 ms 2024-11-08T10:22:01,379 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:22:01,379 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 68acff9b3e69cb0077e2babe1fe22e73 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:01,394 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/.tmp/testFamily/0af60c42eeb2474c96353af5d0889016 is 61, key is row-0/testFamily:testQualifier/1731061151888/Put/seqid=0 2024-11-08T10:22:01,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741914_1090 (size=5049) 2024-11-08T10:22:01,470 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:01,671 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, 68acff9b3e69cb0077e2babe1fe22e73, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:01,799 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/.tmp/testFamily/0af60c42eeb2474c96353af5d0889016 2024-11-08T10:22:01,806 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/.tmp/testFamily/0af60c42eeb2474c96353af5d0889016 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/testFamily/0af60c42eeb2474c96353af5d0889016 2024-11-08T10:22:01,812 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/testFamily/0af60c42eeb2474c96353af5d0889016, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:01,812 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 68acff9b3e69cb0077e2babe1fe22e73 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:01,816 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testRetryOutOfOrderScannerNextException_0_-false/68acff9b3e69cb0077e2babe1fe22e73/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:01,817 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:22:01,817 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 68acff9b3e69cb0077e2babe1fe22e73: 2024-11-08T10:22:01,817 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731061150858.68acff9b3e69cb0077e2babe1fe22e73. 2024-11-08T10:22:01,817 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 517b4604f72f9dd7b55701f7411e363a, disabling compactions & flushes 2024-11-08T10:22:01,817 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:22:01,817 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:22:01,817 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. after waiting 0 ms 2024-11-08T10:22:01,817 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:22:01,817 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 517b4604f72f9dd7b55701f7411e363a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:01,832 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/.tmp/testFamily/d807debe31934b4e9955f41de2d5cae8 is 61, key is row-0/testFamily:testQualifier/1731061309826/Put/seqid=0 2024-11-08T10:22:01,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741915_1091 (size=5049) 2024-11-08T10:22:01,871 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 9 regions to close 2024-11-08T10:22:01,871 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1603): Online Regions={517b4604f72f9dd7b55701f7411e363a=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a., c39a56b147dd6b099a315e1ebc0190be=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be., a45a033254af53ee4f828971c00bd5c3=hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3., d1ef8d97072ec94ee606727c3fa0252e=testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e., f6d402c0ed49fa60db33e36ec6f7dc26=testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26., c98f706b3d1970465cfb9fe302b92eeb=hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb., 645e70ee8fcf32e5b76639c429a17b6d=testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d., eb62f731f61df3966926bac4cef0c26e=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e., a4af48f1a9f782520909de9bd94b26d2=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.} 2024-11-08T10:22:01,871 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:02,071 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 517b4604f72f9dd7b55701f7411e363a, 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:02,237 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/.tmp/testFamily/d807debe31934b4e9955f41de2d5cae8 2024-11-08T10:22:02,244 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/.tmp/testFamily/d807debe31934b4e9955f41de2d5cae8 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/testFamily/d807debe31934b4e9955f41de2d5cae8 2024-11-08T10:22:02,250 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/testFamily/d807debe31934b4e9955f41de2d5cae8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:02,251 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 517b4604f72f9dd7b55701f7411e363a in 434ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:02,255 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/517b4604f72f9dd7b55701f7411e363a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:02,255 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:22:02,255 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 517b4604f72f9dd7b55701f7411e363a: 2024-11-08T10:22:02,255 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731061308810.517b4604f72f9dd7b55701f7411e363a. 2024-11-08T10:22:02,255 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c39a56b147dd6b099a315e1ebc0190be, disabling compactions & flushes 2024-11-08T10:22:02,255 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:22:02,256 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:22:02,256 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. after waiting 0 ms 2024-11-08T10:22:02,256 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:22:02,256 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c39a56b147dd6b099a315e1ebc0190be 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:02,271 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/.tmp/testFamily/43b17da54a2b45aea63a7ec3957984d7 is 61, key is row-0/testFamily:testQualifier/1731061287706/Put/seqid=0 2024-11-08T10:22:02,272 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:02,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741916_1092 (size=5049) 2024-11-08T10:22:02,472 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:02,672 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c39a56b147dd6b099a315e1ebc0190be, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:02,676 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/.tmp/testFamily/43b17da54a2b45aea63a7ec3957984d7 2024-11-08T10:22:02,682 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/.tmp/testFamily/43b17da54a2b45aea63a7ec3957984d7 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/testFamily/43b17da54a2b45aea63a7ec3957984d7 2024-11-08T10:22:02,688 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/testFamily/43b17da54a2b45aea63a7ec3957984d7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:02,689 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c39a56b147dd6b099a315e1ebc0190be in 433ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:02,693 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c39a56b147dd6b099a315e1ebc0190be/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:02,693 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:22:02,693 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c39a56b147dd6b099a315e1ebc0190be: 2024-11-08T10:22:02,693 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731061284688.c39a56b147dd6b099a315e1ebc0190be. 2024-11-08T10:22:02,693 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a45a033254af53ee4f828971c00bd5c3, disabling compactions & flushes 2024-11-08T10:22:02,693 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:22:02,693 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:22:02,694 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. after waiting 0 ms 2024-11-08T10:22:02,694 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:22:02,694 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a45a033254af53ee4f828971c00bd5c3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:02,709 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/.tmp/testFamily/4f790d68bbd0491698055a4cefc43818 is 61, key is row-0/testFamily:testQualifier/1731061299770/Put/seqid=0 2024-11-08T10:22:02,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741917_1093 (size=5049) 2024-11-08T10:22:02,872 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 7 regions to close 2024-11-08T10:22:02,872 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1603): Online Regions={a45a033254af53ee4f828971c00bd5c3=hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3., d1ef8d97072ec94ee606727c3fa0252e=testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e., f6d402c0ed49fa60db33e36ec6f7dc26=testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26., c98f706b3d1970465cfb9fe302b92eeb=hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb., 645e70ee8fcf32e5b76639c429a17b6d=testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d., eb62f731f61df3966926bac4cef0c26e=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e., a4af48f1a9f782520909de9bd94b26d2=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.} 2024-11-08T10:22:02,872 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:03,073 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a45a033254af53ee4f828971c00bd5c3, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:03,114 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/.tmp/testFamily/4f790d68bbd0491698055a4cefc43818 2024-11-08T10:22:03,120 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/.tmp/testFamily/4f790d68bbd0491698055a4cefc43818 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/testFamily/4f790d68bbd0491698055a4cefc43818 2024-11-08T10:22:03,126 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/testFamily/4f790d68bbd0491698055a4cefc43818, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:03,127 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a45a033254af53ee4f828971c00bd5c3 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:03,130 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNext_1_-true/a45a033254af53ee4f828971c00bd5c3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:03,131 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:22:03,131 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a45a033254af53ee4f828971c00bd5c3: 2024-11-08T10:22:03,131 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731061296753.a45a033254af53ee4f828971c00bd5c3. 2024-11-08T10:22:03,131 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d1ef8d97072ec94ee606727c3fa0252e, disabling compactions & flushes 2024-11-08T10:22:03,131 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:22:03,131 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:22:03,131 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. after waiting 0 ms 2024-11-08T10:22:03,131 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:22:03,131 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d1ef8d97072ec94ee606727c3fa0252e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:03,147 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/.tmp/testFamily/2b5372c04d3245e5a1066f63d6bb617f is 61, key is row-0/testFamily:testQualifier/1731061152994/Put/seqid=0 2024-11-08T10:22:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741918_1094 (size=5049) 2024-11-08T10:22:03,273 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:03,473 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, d1ef8d97072ec94ee606727c3fa0252e, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:03,551 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/.tmp/testFamily/2b5372c04d3245e5a1066f63d6bb617f 2024-11-08T10:22:03,558 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/.tmp/testFamily/2b5372c04d3245e5a1066f63d6bb617f as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/testFamily/2b5372c04d3245e5a1066f63d6bb617f 2024-11-08T10:22:03,564 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/testFamily/2b5372c04d3245e5a1066f63d6bb617f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:03,565 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d1ef8d97072ec94ee606727c3fa0252e in 434ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:03,569 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_0_-false/d1ef8d97072ec94ee606727c3fa0252e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:03,569 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:22:03,569 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d1ef8d97072ec94ee606727c3fa0252e: 2024-11-08T10:22:03,570 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731061151971.d1ef8d97072ec94ee606727c3fa0252e. 2024-11-08T10:22:03,570 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f6d402c0ed49fa60db33e36ec6f7dc26, disabling compactions & flushes 2024-11-08T10:22:03,570 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:22:03,570 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:22:03,570 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. after waiting 0 ms 2024-11-08T10:22:03,570 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:22:03,570 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f6d402c0ed49fa60db33e36ec6f7dc26 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:03,590 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/.tmp/testFamily/0dc7e24118194317ac9dcdd0444d167f is 61, key is row-0/testFamily:testQualifier/1731061240263/Put/seqid=0 2024-11-08T10:22:03,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741919_1095 (size=5049) 2024-11-08T10:22:03,673 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:03,874 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 5 regions to close 2024-11-08T10:22:03,874 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1603): Online Regions={f6d402c0ed49fa60db33e36ec6f7dc26=testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26., c98f706b3d1970465cfb9fe302b92eeb=hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb., 645e70ee8fcf32e5b76639c429a17b6d=testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d., eb62f731f61df3966926bac4cef0c26e=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e., a4af48f1a9f782520909de9bd94b26d2=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.} 2024-11-08T10:22:03,874 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, eb62f731f61df3966926bac4cef0c26e, f6d402c0ed49fa60db33e36ec6f7dc26 2024-11-08T10:22:03,995 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/.tmp/testFamily/0dc7e24118194317ac9dcdd0444d167f 2024-11-08T10:22:04,001 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/.tmp/testFamily/0dc7e24118194317ac9dcdd0444d167f as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/testFamily/0dc7e24118194317ac9dcdd0444d167f 2024-11-08T10:22:04,007 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/testFamily/0dc7e24118194317ac9dcdd0444d167f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:04,008 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f6d402c0ed49fa60db33e36ec6f7dc26 in 438ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:04,011 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnRenewLease_1_-true/f6d402c0ed49fa60db33e36ec6f7dc26/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:04,012 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:22:04,012 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f6d402c0ed49fa60db33e36ec6f7dc26: 2024-11-08T10:22:04,012 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731061237247.f6d402c0ed49fa60db33e36ec6f7dc26. 2024-11-08T10:22:04,012 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c98f706b3d1970465cfb9fe302b92eeb, disabling compactions & flushes 2024-11-08T10:22:04,012 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:22:04,012 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:22:04,012 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. after waiting 0 ms 2024-11-08T10:22:04,012 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:22:04,012 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c98f706b3d1970465cfb9fe302b92eeb 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-08T10:22:04,029 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/.tmp/info/7877437a696c43e5a4f7d9bfeb4feeb6 is 45, key is default/info:d/1731061128971/Put/seqid=0 2024-11-08T10:22:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741920_1096 (size=5037) 2024-11-08T10:22:04,074 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:22:04,274 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, c98f706b3d1970465cfb9fe302b92eeb, eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:22:04,434 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/.tmp/info/7877437a696c43e5a4f7d9bfeb4feeb6 2024-11-08T10:22:04,440 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/.tmp/info/7877437a696c43e5a4f7d9bfeb4feeb6 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/info/7877437a696c43e5a4f7d9bfeb4feeb6 2024-11-08T10:22:04,446 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/info/7877437a696c43e5a4f7d9bfeb4feeb6, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T10:22:04,447 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for c98f706b3d1970465cfb9fe302b92eeb in 435ms, sequenceid=6, compaction requested=false 2024-11-08T10:22:04,450 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/namespace/c98f706b3d1970465cfb9fe302b92eeb/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T10:22:04,451 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:22:04,451 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c98f706b3d1970465cfb9fe302b92eeb: 2024-11-08T10:22:04,451 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731061127866.c98f706b3d1970465cfb9fe302b92eeb. 2024-11-08T10:22:04,451 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 645e70ee8fcf32e5b76639c429a17b6d, disabling compactions & flushes 2024-11-08T10:22:04,451 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:22:04,451 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:22:04,451 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. after waiting 0 ms 2024-11-08T10:22:04,451 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:22:04,451 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 645e70ee8fcf32e5b76639c429a17b6d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:04,467 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/.tmp/testFamily/f97f9469a68849208ffbba3bf7893e49 is 61, key is row-0/testFamily:testQualifier/1731061160196/Put/seqid=0 2024-11-08T10:22:04,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741921_1097 (size=5049) 2024-11-08T10:22:04,474 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:22:04,675 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:22:04,872 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/.tmp/testFamily/f97f9469a68849208ffbba3bf7893e49 2024-11-08T10:22:04,875 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-11-08T10:22:04,875 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1603): Online Regions={645e70ee8fcf32e5b76639c429a17b6d=testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d., eb62f731f61df3966926bac4cef0c26e=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e., a4af48f1a9f782520909de9bd94b26d2=hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2.} 2024-11-08T10:22:04,875 DEBUG [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1629): Waiting on 645e70ee8fcf32e5b76639c429a17b6d, a4af48f1a9f782520909de9bd94b26d2, eb62f731f61df3966926bac4cef0c26e 2024-11-08T10:22:04,878 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/.tmp/testFamily/f97f9469a68849208ffbba3bf7893e49 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/testFamily/f97f9469a68849208ffbba3bf7893e49 2024-11-08T10:22:04,884 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/testFamily/f97f9469a68849208ffbba3bf7893e49, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:04,885 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 645e70ee8fcf32e5b76639c429a17b6d in 434ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:04,890 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/default/testNormalScanTimeoutOnNextAsync_0_-false/645e70ee8fcf32e5b76639c429a17b6d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:04,891 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:22:04,891 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 645e70ee8fcf32e5b76639c429a17b6d: 2024-11-08T10:22:04,891 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731061159171.645e70ee8fcf32e5b76639c429a17b6d. 2024-11-08T10:22:04,891 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing eb62f731f61df3966926bac4cef0c26e, disabling compactions & flushes 2024-11-08T10:22:04,891 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:22:04,891 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:22:04,891 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. after waiting 0 ms 2024-11-08T10:22:04,891 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:22:04,891 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing eb62f731f61df3966926bac4cef0c26e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:04,907 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/.tmp/testFamily/cd5cc15d9a6e49aa9f9604f26edddbaa is 61, key is row-0/testFamily:testQualifier/1731061141794/Put/seqid=0 2024-11-08T10:22:04,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741922_1098 (size=5049) 2024-11-08T10:22:04,913 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/.tmp/testFamily/cd5cc15d9a6e49aa9f9604f26edddbaa 2024-11-08T10:22:04,919 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/.tmp/testFamily/cd5cc15d9a6e49aa9f9604f26edddbaa as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/testFamily/cd5cc15d9a6e49aa9f9604f26edddbaa 2024-11-08T10:22:04,924 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/testFamily/cd5cc15d9a6e49aa9f9604f26edddbaa, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:04,925 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb62f731f61df3966926bac4cef0c26e in 34ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:04,929 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/eb62f731f61df3966926bac4cef0c26e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:04,930 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:22:04,930 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for eb62f731f61df3966926bac4cef0c26e: 2024-11-08T10:22:04,930 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731061140775.eb62f731f61df3966926bac4cef0c26e. 2024-11-08T10:22:04,930 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a4af48f1a9f782520909de9bd94b26d2, disabling compactions & flushes 2024-11-08T10:22:04,930 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:22:04,930 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:22:04,930 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. after waiting 0 ms 2024-11-08T10:22:04,930 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:22:04,930 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a4af48f1a9f782520909de9bd94b26d2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T10:22:04,945 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/.tmp/testFamily/8401c9b568bb45b39849c1e5a407f7d1 is 61, key is row-0/testFamily:testQualifier/1731061277632/Put/seqid=0 2024-11-08T10:22:04,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741923_1099 (size=5049) 2024-11-08T10:22:04,950 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/.tmp/testFamily/8401c9b568bb45b39849c1e5a407f7d1 2024-11-08T10:22:04,956 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/.tmp/testFamily/8401c9b568bb45b39849c1e5a407f7d1 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/testFamily/8401c9b568bb45b39849c1e5a407f7d1 2024-11-08T10:22:04,962 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/testFamily/8401c9b568bb45b39849c1e5a407f7d1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T10:22:04,963 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a4af48f1a9f782520909de9bd94b26d2 in 33ms, sequenceid=8, compaction requested=false 2024-11-08T10:22:04,968 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a4af48f1a9f782520909de9bd94b26d2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T10:22:04,969 INFO [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:22:04,969 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a4af48f1a9f782520909de9bd94b26d2: 2024-11-08T10:22:04,969 DEBUG [RS_CLOSE_REGION-regionserver/2db67accb017:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731061274610.a4af48f1a9f782520909de9bd94b26d2. 2024-11-08T10:22:05,075 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1250): stopping server 2db67accb017,37913,1731061095589; all regions closed. 2024-11-08T10:22:05,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741834_1010 (size=56731) 2024-11-08T10:22:05,085 DEBUG [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/oldWALs 2024-11-08T10:22:05,085 INFO [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 2db67accb017%2C37913%2C1731061095589.meta:.meta(num 1731061127542) 2024-11-08T10:22:05,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741833_1009 (size=60562) 2024-11-08T10:22:05,089 DEBUG [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/oldWALs 2024-11-08T10:22:05,089 INFO [RS:0;2db67accb017:37913 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 2db67accb017%2C37913%2C1731061095589:(num 1731061127214) 2024-11-08T10:22:05,089 DEBUG [RS:0;2db67accb017:37913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:22:05,090 INFO [RS:0;2db67accb017:37913 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:22:05,090 INFO [RS:0;2db67accb017:37913 {}] hbase.ChoreService(370): Chore service for: regionserver/2db67accb017:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T10:22:05,090 INFO [regionserver/2db67accb017:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T10:22:05,091 INFO [RS:0;2db67accb017:37913 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:37913 2024-11-08T10:22:05,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2db67accb017,37913,1731061095589 2024-11-08T10:22:05,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:22:05,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2db67accb017,37913,1731061095589] 2024-11-08T10:22:05,096 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 2db67accb017,37913,1731061095589; numProcessing=1 2024-11-08T10:22:05,098 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/2db67accb017,37913,1731061095589 already deleted, retry=false 2024-11-08T10:22:05,098 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 2db67accb017,37913,1731061095589 expired; onlineServers=0 2024-11-08T10:22:05,098 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '2db67accb017,40511,1731061094572' ***** 2024-11-08T10:22:05,098 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T10:22:05,098 DEBUG [M:0;2db67accb017:40511 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3160cd8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2db67accb017/172.17.0.2:0 2024-11-08T10:22:05,098 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegionServer(1224): stopping server 2db67accb017,40511,1731061094572 2024-11-08T10:22:05,098 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegionServer(1250): stopping server 2db67accb017,40511,1731061094572; all regions closed. 2024-11-08T10:22:05,098 DEBUG [M:0;2db67accb017:40511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:22:05,098 DEBUG [M:0;2db67accb017:40511 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T10:22:05,098 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T10:22:05,098 DEBUG [M:0;2db67accb017:40511 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T10:22:05,099 DEBUG [master/2db67accb017:0:becomeActiveMaster-HFileCleaner.small.0-1731061096884 {}] cleaner.HFileCleaner(306): Exit Thread[master/2db67accb017:0:becomeActiveMaster-HFileCleaner.small.0-1731061096884,5,FailOnTimeoutGroup] 2024-11-08T10:22:05,099 DEBUG [master/2db67accb017:0:becomeActiveMaster-HFileCleaner.large.0-1731061096882 {}] cleaner.HFileCleaner(306): Exit Thread[master/2db67accb017:0:becomeActiveMaster-HFileCleaner.large.0-1731061096882,5,FailOnTimeoutGroup] 2024-11-08T10:22:05,099 INFO [M:0;2db67accb017:40511 {}] hbase.ChoreService(370): Chore service for: master/2db67accb017:0 had [] on shutdown 2024-11-08T10:22:05,099 DEBUG [M:0;2db67accb017:40511 {}] master.HMaster(1733): Stopping service threads 2024-11-08T10:22:05,099 INFO [M:0;2db67accb017:40511 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T10:22:05,099 ERROR [M:0;2db67accb017:40511 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[LeaseRenewer:jenkins@localhost:46031,5,PEWorkerGroup] 2024-11-08T10:22:05,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T10:22:05,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:22:05,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:22:05,100 INFO [M:0;2db67accb017:40511 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T10:22:05,100 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T10:22:05,101 DEBUG [M:0;2db67accb017:40511 {}] zookeeper.ZKUtil(347): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T10:22:05,101 WARN [M:0;2db67accb017:40511 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T10:22:05,101 INFO [M:0;2db67accb017:40511 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-08T10:22:05,101 INFO [M:0;2db67accb017:40511 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T10:22:05,101 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T10:22:05,101 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:22:05,101 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:22:05,101 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T10:22:05,101 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:22:05,102 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.59 KB heapSize=520.21 KB 2024-11-08T10:22:05,123 INFO [regionserver/2db67accb017:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:22:05,126 DEBUG [M:0;2db67accb017:40511 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d59d477be3d142b3a1998d38cbaff7b8 is 82, key is hbase:meta,,1/info:regioninfo/1731061127729/Put/seqid=0 2024-11-08T10:22:05,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741924_1100 (size=5672) 2024-11-08T10:22:05,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:22:05,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37913-0x10187638bc40001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:22:05,196 INFO [RS:0;2db67accb017:37913 {}] regionserver.HRegionServer(1307): Exiting; stopping=2db67accb017,37913,1731061095589; zookeeper connection closed. 2024-11-08T10:22:05,196 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9bc43fc {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9bc43fc 2024-11-08T10:22:05,197 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T10:22:05,532 INFO [M:0;2db67accb017:40511 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d59d477be3d142b3a1998d38cbaff7b8 2024-11-08T10:22:05,555 DEBUG [M:0;2db67accb017:40511 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5435992bc29410bacfe240cbc63f80e is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731061129959/Put/seqid=0 2024-11-08T10:22:05,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741925_1101 (size=29526) 2024-11-08T10:22:05,561 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T10:22:05,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T10:22:05,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T10:22:05,962 INFO [M:0;2db67accb017:40511 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=448.03 KB at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5435992bc29410bacfe240cbc63f80e 2024-11-08T10:22:05,968 INFO [M:0;2db67accb017:40511 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a5435992bc29410bacfe240cbc63f80e 2024-11-08T10:22:05,984 DEBUG [M:0;2db67accb017:40511 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03775b84c5b64016bac5e104ea8310c0 is 69, key is 2db67accb017,37913,1731061095589/rs:state/1731061096937/Put/seqid=0 2024-11-08T10:22:05,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741926_1102 (size=5156) 2024-11-08T10:22:05,988 INFO [M:0;2db67accb017:40511 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03775b84c5b64016bac5e104ea8310c0 2024-11-08T10:22:05,994 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d59d477be3d142b3a1998d38cbaff7b8 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d59d477be3d142b3a1998d38cbaff7b8 2024-11-08T10:22:05,999 INFO [M:0;2db67accb017:40511 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d59d477be3d142b3a1998d38cbaff7b8, entries=8, sequenceid=832, filesize=5.5 K 2024-11-08T10:22:06,000 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5435992bc29410bacfe240cbc63f80e as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a5435992bc29410bacfe240cbc63f80e 2024-11-08T10:22:06,005 INFO [M:0;2db67accb017:40511 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a5435992bc29410bacfe240cbc63f80e 2024-11-08T10:22:06,005 INFO [M:0;2db67accb017:40511 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a5435992bc29410bacfe240cbc63f80e, entries=92, sequenceid=832, filesize=28.8 K 2024-11-08T10:22:06,006 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03775b84c5b64016bac5e104ea8310c0 as hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03775b84c5b64016bac5e104ea8310c0 2024-11-08T10:22:06,011 INFO [M:0;2db67accb017:40511 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46031/user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03775b84c5b64016bac5e104ea8310c0, entries=1, sequenceid=832, filesize=5.0 K 2024-11-08T10:22:06,012 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.59 KB/459353, heapSize ~519.91 KB/532392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 911ms, sequenceid=832, compaction requested=false 2024-11-08T10:22:06,014 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:22:06,014 DEBUG [M:0;2db67accb017:40511 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T10:22:06,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37041 is added to blk_1073741830_1006 (size=514352) 2024-11-08T10:22:06,017 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/785994ea-5a43-764d-a9c5-d51800087d01/MasterData/WALs/2db67accb017,40511,1731061094572/2db67accb017%2C40511%2C1731061094572.1731061096230 not finished, retry = 0 2024-11-08T10:22:06,118 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T10:22:06,118 INFO [M:0;2db67accb017:40511 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-08T10:22:06,118 INFO [M:0;2db67accb017:40511 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40511 2024-11-08T10:22:06,120 DEBUG [M:0;2db67accb017:40511 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/2db67accb017,40511,1731061094572 already deleted, retry=false 2024-11-08T10:22:06,222 INFO [M:0;2db67accb017:40511 {}] regionserver.HRegionServer(1307): Exiting; stopping=2db67accb017,40511,1731061094572; zookeeper connection closed. 2024-11-08T10:22:06,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:22:06,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40511-0x10187638bc40000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:22:06,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ae43da4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T10:22:06,230 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@540cee33{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T10:22:06,230 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T10:22:06,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c8914a8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T10:22:06,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ed11b40{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/hadoop.log.dir/,STOPPED} 2024-11-08T10:22:06,232 WARN [BP-1631354101-172.17.0.2-1731061091046 heartbeating to localhost/127.0.0.1:46031 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T10:22:06,232 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T10:22:06,232 WARN [BP-1631354101-172.17.0.2-1731061091046 heartbeating to localhost/127.0.0.1:46031 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1631354101-172.17.0.2-1731061091046 (Datanode Uuid 9f7de0e8-95e7-42c9-b2b6-c2442a647b59) service to localhost/127.0.0.1:46031 2024-11-08T10:22:06,232 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T10:22:06,234 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/dfs/data/data1/current/BP-1631354101-172.17.0.2-1731061091046 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T10:22:06,234 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/cluster_d36a9e5c-0a3f-65c0-f5e4-442aad84f92b/dfs/data/data2/current/BP-1631354101-172.17.0.2-1731061091046 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T10:22:06,235 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T10:22:06,242 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1af42a04{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T10:22:06,242 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8a3c7b8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T10:22:06,242 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T10:22:06,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d3a9ea5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T10:22:06,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c68af8f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/76a0ba66-6268-2c2c-a24d-12505cd6c2a7/hadoop.log.dir/,STOPPED} 2024-11-08T10:22:06,252 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-08T10:22:06,285 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down